diff --git a/lib/python3.12/site-packages/_multiprocess/__init__.py b/lib/python3.12/site-packages/_multiprocess/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..085958182e2246471e6d6f655ff00b6e37e3e99a --- /dev/null +++ b/lib/python3.12/site-packages/_multiprocess/__init__.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python +# +# Author: Mike McKerns (mmckerns @caltech and @uqfoundation) +# Copyright (c) 2022-2024 The Uncertainty Quantification Foundation. +# License: 3-clause BSD. The full license text is available at: +# - https://github.com/uqfoundation/multiprocess/blob/master/LICENSE + +from _multiprocessing import * diff --git a/lib/python3.12/site-packages/_multiprocess/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/_multiprocess/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a577e47bd2fe3ce37f303509144365e4c7f3abb8 Binary files /dev/null and b/lib/python3.12/site-packages/_multiprocess/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a29ba1a613bad61021cb17f519faa15a59e1c677 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/big_modeling.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/big_modeling.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c47f7a63d721814f8f8c69ab353bc9c268e8b04b Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/big_modeling.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/checkpointing.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/checkpointing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..037e789cd778b5cece7eb66e23f17a0d339eee95 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/checkpointing.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/data_loader.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/data_loader.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20231706ba4ac856bfd965e9f9e4949040afa14f Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/data_loader.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/hooks.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/hooks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8ca0d6689d485a7935b604ac3e8dda3bf87616b Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/hooks.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/inference.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/inference.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0efead92f8e6fd371107afc540ace56229990808 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/inference.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/launchers.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/launchers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..853d976f7f277f4a8a47371bf69313651f15d06e Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/launchers.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/local_sgd.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/local_sgd.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..adf97662ebc3d24420a93fb043b46289be32899b Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/local_sgd.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/logging.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/logging.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7dda6ec15fcb8d679aa667735a9a6b9ef51169c6 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/logging.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/memory_utils.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/memory_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..79392eb981cdbeb6c89f91fde265a643dda7afa5 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/memory_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/optimizer.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/optimizer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..678942721ee09183c82a5bcd44c659d9021b6242 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/optimizer.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/scheduler.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/scheduler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..04151ee1995fcea1fd8943388e5832b630fcadd3 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/scheduler.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/state.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/state.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..641a4d7dd9d7d3703a6eee8fe993d315fb9c3e5b Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/state.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/__pycache__/tracking.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/__pycache__/tracking.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd6f78746aa3886b548b84fcbc86776049493199 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/__pycache__/tracking.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/test_utils/scripts/__pycache__/test_cli.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/test_utils/scripts/__pycache__/test_cli.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0e063ffabe92e972c66134d2c7bbafa076b982a8 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/test_utils/scripts/__pycache__/test_cli.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/accelerate/test_utils/scripts/__pycache__/test_script.cpython-312.pyc b/lib/python3.12/site-packages/accelerate/test_utils/scripts/__pycache__/test_script.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..838c8438a9c1803e2394160d64e28ef461cfb6f3 Binary files /dev/null and b/lib/python3.12/site-packages/accelerate/test_utils/scripts/__pycache__/test_script.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/blinker/__init__.py b/lib/python3.12/site-packages/blinker/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1772fa4a543b0288f03d60050f222ed64a83ece0 --- /dev/null +++ b/lib/python3.12/site-packages/blinker/__init__.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from .base import ANY +from .base import default_namespace +from .base import NamedSignal +from .base import Namespace +from .base import Signal +from .base import signal + +__all__ = [ + "ANY", + "default_namespace", + "NamedSignal", + "Namespace", + "Signal", + "signal", +] diff --git a/lib/python3.12/site-packages/blinker/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/blinker/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1778d1a56a1b1d2cff8d1e6313f015313860be44 Binary files /dev/null and b/lib/python3.12/site-packages/blinker/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/blinker/__pycache__/_utilities.cpython-312.pyc b/lib/python3.12/site-packages/blinker/__pycache__/_utilities.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa9b1ee79bb133080bbb93a9f3a262fbd61aa769 Binary files /dev/null and b/lib/python3.12/site-packages/blinker/__pycache__/_utilities.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/blinker/__pycache__/base.cpython-312.pyc b/lib/python3.12/site-packages/blinker/__pycache__/base.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..86f04684fc217d64ad9e57847e1c4ca025aea00b Binary files /dev/null and b/lib/python3.12/site-packages/blinker/__pycache__/base.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/blinker/_utilities.py b/lib/python3.12/site-packages/blinker/_utilities.py new file mode 100644 index 0000000000000000000000000000000000000000..000c902a2564d2d4a551edb646a9d654d9e7041b --- /dev/null +++ b/lib/python3.12/site-packages/blinker/_utilities.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import collections.abc as c +import inspect +import typing as t +from weakref import ref +from weakref import WeakMethod + +T = t.TypeVar("T") + + +class Symbol: + """A constant symbol, nicer than ``object()``. Repeated calls return the + same instance. + + >>> Symbol('foo') is Symbol('foo') + True + >>> Symbol('foo') + foo + """ + + symbols: t.ClassVar[dict[str, Symbol]] = {} + + def __new__(cls, name: str) -> Symbol: + if name in cls.symbols: + return cls.symbols[name] + + obj = super().__new__(cls) + cls.symbols[name] = obj + return obj + + def __init__(self, name: str) -> None: + self.name = name + + def __repr__(self) -> str: + return self.name + + def __getnewargs__(self) -> tuple[t.Any, ...]: + return (self.name,) + + +def make_id(obj: object) -> c.Hashable: + """Get a stable identifier for a receiver or sender, to be used as a dict + key or in a set. + """ + if inspect.ismethod(obj): + # The id of a bound method is not stable, but the id of the unbound + # function and instance are. + return id(obj.__func__), id(obj.__self__) + + if isinstance(obj, (str, int)): + # Instances with the same value always compare equal and have the same + # hash, even if the id may change. + return obj + + # Assume other types are not hashable but will always be the same instance. + return id(obj) + + +def make_ref(obj: T, callback: c.Callable[[ref[T]], None] | None = None) -> ref[T]: + if inspect.ismethod(obj): + return WeakMethod(obj, callback) # type: ignore[arg-type, return-value] + + return ref(obj, callback) diff --git a/lib/python3.12/site-packages/blinker/base.py b/lib/python3.12/site-packages/blinker/base.py new file mode 100644 index 0000000000000000000000000000000000000000..d051b94a32f5eecd9f32853d6eaebcca6c7133f6 --- /dev/null +++ b/lib/python3.12/site-packages/blinker/base.py @@ -0,0 +1,512 @@ +from __future__ import annotations + +import collections.abc as c +import sys +import typing as t +import weakref +from collections import defaultdict +from contextlib import contextmanager +from functools import cached_property +from inspect import iscoroutinefunction + +from ._utilities import make_id +from ._utilities import make_ref +from ._utilities import Symbol + +F = t.TypeVar("F", bound=c.Callable[..., t.Any]) + +ANY = Symbol("ANY") +"""Symbol for "any sender".""" + +ANY_ID = 0 + + +class Signal: + """A notification emitter. + + :param doc: The docstring for the signal. + """ + + ANY = ANY + """An alias for the :data:`~blinker.ANY` sender symbol.""" + + set_class: type[set[t.Any]] = set + """The set class to use for tracking connected receivers and senders. + Python's ``set`` is unordered. If receivers must be dispatched in the order + they were connected, an ordered set implementation can be used. + + .. versionadded:: 1.7 + """ + + @cached_property + def receiver_connected(self) -> Signal: + """Emitted at the end of each :meth:`connect` call. + + The signal sender is the signal instance, and the :meth:`connect` + arguments are passed through: ``receiver``, ``sender``, and ``weak``. + + .. versionadded:: 1.2 + """ + return Signal(doc="Emitted after a receiver connects.") + + @cached_property + def receiver_disconnected(self) -> Signal: + """Emitted at the end of each :meth:`disconnect` call. + + The sender is the signal instance, and the :meth:`disconnect` arguments + are passed through: ``receiver`` and ``sender``. + + This signal is emitted **only** when :meth:`disconnect` is called + explicitly. This signal cannot be emitted by an automatic disconnect + when a weakly referenced receiver or sender goes out of scope, as the + instance is no longer be available to be used as the sender for this + signal. + + An alternative approach is available by subscribing to + :attr:`receiver_connected` and setting up a custom weakref cleanup + callback on weak receivers and senders. + + .. versionadded:: 1.2 + """ + return Signal(doc="Emitted after a receiver disconnects.") + + def __init__(self, doc: str | None = None) -> None: + if doc: + self.__doc__ = doc + + self.receivers: dict[ + t.Any, weakref.ref[c.Callable[..., t.Any]] | c.Callable[..., t.Any] + ] = {} + """The map of connected receivers. Useful to quickly check if any + receivers are connected to the signal: ``if s.receivers:``. The + structure and data is not part of the public API, but checking its + boolean value is. + """ + + self.is_muted: bool = False + self._by_receiver: dict[t.Any, set[t.Any]] = defaultdict(self.set_class) + self._by_sender: dict[t.Any, set[t.Any]] = defaultdict(self.set_class) + self._weak_senders: dict[t.Any, weakref.ref[t.Any]] = {} + + def connect(self, receiver: F, sender: t.Any = ANY, weak: bool = True) -> F: + """Connect ``receiver`` to be called when the signal is sent by + ``sender``. + + :param receiver: The callable to call when :meth:`send` is called with + the given ``sender``, passing ``sender`` as a positional argument + along with any extra keyword arguments. + :param sender: Any object or :data:`ANY`. ``receiver`` will only be + called when :meth:`send` is called with this sender. If ``ANY``, the + receiver will be called for any sender. A receiver may be connected + to multiple senders by calling :meth:`connect` multiple times. + :param weak: Track the receiver with a :mod:`weakref`. The receiver will + be automatically disconnected when it is garbage collected. When + connecting a receiver defined within a function, set to ``False``, + otherwise it will be disconnected when the function scope ends. + """ + receiver_id = make_id(receiver) + sender_id = ANY_ID if sender is ANY else make_id(sender) + + if weak: + self.receivers[receiver_id] = make_ref( + receiver, self._make_cleanup_receiver(receiver_id) + ) + else: + self.receivers[receiver_id] = receiver + + self._by_sender[sender_id].add(receiver_id) + self._by_receiver[receiver_id].add(sender_id) + + if sender is not ANY and sender_id not in self._weak_senders: + # store a cleanup for weakref-able senders + try: + self._weak_senders[sender_id] = make_ref( + sender, self._make_cleanup_sender(sender_id) + ) + except TypeError: + pass + + if "receiver_connected" in self.__dict__ and self.receiver_connected.receivers: + try: + self.receiver_connected.send( + self, receiver=receiver, sender=sender, weak=weak + ) + except TypeError: + # TODO no explanation or test for this + self.disconnect(receiver, sender) + raise + + return receiver + + def connect_via(self, sender: t.Any, weak: bool = False) -> c.Callable[[F], F]: + """Connect the decorated function to be called when the signal is sent + by ``sender``. + + The decorated function will be called when :meth:`send` is called with + the given ``sender``, passing ``sender`` as a positional argument along + with any extra keyword arguments. + + :param sender: Any object or :data:`ANY`. ``receiver`` will only be + called when :meth:`send` is called with this sender. If ``ANY``, the + receiver will be called for any sender. A receiver may be connected + to multiple senders by calling :meth:`connect` multiple times. + :param weak: Track the receiver with a :mod:`weakref`. The receiver will + be automatically disconnected when it is garbage collected. When + connecting a receiver defined within a function, set to ``False``, + otherwise it will be disconnected when the function scope ends.= + + .. versionadded:: 1.1 + """ + + def decorator(fn: F) -> F: + self.connect(fn, sender, weak) + return fn + + return decorator + + @contextmanager + def connected_to( + self, receiver: c.Callable[..., t.Any], sender: t.Any = ANY + ) -> c.Generator[None, None, None]: + """A context manager that temporarily connects ``receiver`` to the + signal while a ``with`` block executes. When the block exits, the + receiver is disconnected. Useful for tests. + + :param receiver: The callable to call when :meth:`send` is called with + the given ``sender``, passing ``sender`` as a positional argument + along with any extra keyword arguments. + :param sender: Any object or :data:`ANY`. ``receiver`` will only be + called when :meth:`send` is called with this sender. If ``ANY``, the + receiver will be called for any sender. + + .. versionadded:: 1.1 + """ + self.connect(receiver, sender=sender, weak=False) + + try: + yield None + finally: + self.disconnect(receiver) + + @contextmanager + def muted(self) -> c.Generator[None, None, None]: + """A context manager that temporarily disables the signal. No receivers + will be called if the signal is sent, until the ``with`` block exits. + Useful for tests. + """ + self.is_muted = True + + try: + yield None + finally: + self.is_muted = False + + def send( + self, + sender: t.Any | None = None, + /, + *, + _async_wrapper: c.Callable[ + [c.Callable[..., c.Coroutine[t.Any, t.Any, t.Any]]], c.Callable[..., t.Any] + ] + | None = None, + **kwargs: t.Any, + ) -> list[tuple[c.Callable[..., t.Any], t.Any]]: + """Call all receivers that are connected to the given ``sender`` + or :data:`ANY`. Each receiver is called with ``sender`` as a positional + argument along with any extra keyword arguments. Return a list of + ``(receiver, return value)`` tuples. + + The order receivers are called is undefined, but can be influenced by + setting :attr:`set_class`. + + If a receiver raises an exception, that exception will propagate up. + This makes debugging straightforward, with an assumption that correctly + implemented receivers will not raise. + + :param sender: Call receivers connected to this sender, in addition to + those connected to :data:`ANY`. + :param _async_wrapper: Will be called on any receivers that are async + coroutines to turn them into sync callables. For example, could run + the receiver with an event loop. + :param kwargs: Extra keyword arguments to pass to each receiver. + + .. versionchanged:: 1.7 + Added the ``_async_wrapper`` argument. + """ + if self.is_muted: + return [] + + results = [] + + for receiver in self.receivers_for(sender): + if iscoroutinefunction(receiver): + if _async_wrapper is None: + raise RuntimeError("Cannot send to a coroutine function.") + + result = _async_wrapper(receiver)(sender, **kwargs) + else: + result = receiver(sender, **kwargs) + + results.append((receiver, result)) + + return results + + async def send_async( + self, + sender: t.Any | None = None, + /, + *, + _sync_wrapper: c.Callable[ + [c.Callable[..., t.Any]], c.Callable[..., c.Coroutine[t.Any, t.Any, t.Any]] + ] + | None = None, + **kwargs: t.Any, + ) -> list[tuple[c.Callable[..., t.Any], t.Any]]: + """Await all receivers that are connected to the given ``sender`` + or :data:`ANY`. Each receiver is called with ``sender`` as a positional + argument along with any extra keyword arguments. Return a list of + ``(receiver, return value)`` tuples. + + The order receivers are called is undefined, but can be influenced by + setting :attr:`set_class`. + + If a receiver raises an exception, that exception will propagate up. + This makes debugging straightforward, with an assumption that correctly + implemented receivers will not raise. + + :param sender: Call receivers connected to this sender, in addition to + those connected to :data:`ANY`. + :param _sync_wrapper: Will be called on any receivers that are sync + callables to turn them into async coroutines. For example, + could call the receiver in a thread. + :param kwargs: Extra keyword arguments to pass to each receiver. + + .. versionadded:: 1.7 + """ + if self.is_muted: + return [] + + results = [] + + for receiver in self.receivers_for(sender): + if not iscoroutinefunction(receiver): + if _sync_wrapper is None: + raise RuntimeError("Cannot send to a non-coroutine function.") + + result = await _sync_wrapper(receiver)(sender, **kwargs) + else: + result = await receiver(sender, **kwargs) + + results.append((receiver, result)) + + return results + + def has_receivers_for(self, sender: t.Any) -> bool: + """Check if there is at least one receiver that will be called with the + given ``sender``. A receiver connected to :data:`ANY` will always be + called, regardless of sender. Does not check if weakly referenced + receivers are still live. See :meth:`receivers_for` for a stronger + search. + + :param sender: Check for receivers connected to this sender, in addition + to those connected to :data:`ANY`. + """ + if not self.receivers: + return False + + if self._by_sender[ANY_ID]: + return True + + if sender is ANY: + return False + + return make_id(sender) in self._by_sender + + def receivers_for( + self, sender: t.Any + ) -> c.Generator[c.Callable[..., t.Any], None, None]: + """Yield each receiver to be called for ``sender``, in addition to those + to be called for :data:`ANY`. Weakly referenced receivers that are not + live will be disconnected and skipped. + + :param sender: Yield receivers connected to this sender, in addition + to those connected to :data:`ANY`. + """ + # TODO: test receivers_for(ANY) + if not self.receivers: + return + + sender_id = make_id(sender) + + if sender_id in self._by_sender: + ids = self._by_sender[ANY_ID] | self._by_sender[sender_id] + else: + ids = self._by_sender[ANY_ID].copy() + + for receiver_id in ids: + receiver = self.receivers.get(receiver_id) + + if receiver is None: + continue + + if isinstance(receiver, weakref.ref): + strong = receiver() + + if strong is None: + self._disconnect(receiver_id, ANY_ID) + continue + + yield strong + else: + yield receiver + + def disconnect(self, receiver: c.Callable[..., t.Any], sender: t.Any = ANY) -> None: + """Disconnect ``receiver`` from being called when the signal is sent by + ``sender``. + + :param receiver: A connected receiver callable. + :param sender: Disconnect from only this sender. By default, disconnect + from all senders. + """ + sender_id: c.Hashable + + if sender is ANY: + sender_id = ANY_ID + else: + sender_id = make_id(sender) + + receiver_id = make_id(receiver) + self._disconnect(receiver_id, sender_id) + + if ( + "receiver_disconnected" in self.__dict__ + and self.receiver_disconnected.receivers + ): + self.receiver_disconnected.send(self, receiver=receiver, sender=sender) + + def _disconnect(self, receiver_id: c.Hashable, sender_id: c.Hashable) -> None: + if sender_id == ANY_ID: + if self._by_receiver.pop(receiver_id, None) is not None: + for bucket in self._by_sender.values(): + bucket.discard(receiver_id) + + self.receivers.pop(receiver_id, None) + else: + self._by_sender[sender_id].discard(receiver_id) + self._by_receiver[receiver_id].discard(sender_id) + + def _make_cleanup_receiver( + self, receiver_id: c.Hashable + ) -> c.Callable[[weakref.ref[c.Callable[..., t.Any]]], None]: + """Create a callback function to disconnect a weakly referenced + receiver when it is garbage collected. + """ + + def cleanup(ref: weakref.ref[c.Callable[..., t.Any]]) -> None: + # If the interpreter is shutting down, disconnecting can result in a + # weird ignored exception. Don't call it in that case. + if not sys.is_finalizing(): + self._disconnect(receiver_id, ANY_ID) + + return cleanup + + def _make_cleanup_sender( + self, sender_id: c.Hashable + ) -> c.Callable[[weakref.ref[t.Any]], None]: + """Create a callback function to disconnect all receivers for a weakly + referenced sender when it is garbage collected. + """ + assert sender_id != ANY_ID + + def cleanup(ref: weakref.ref[t.Any]) -> None: + self._weak_senders.pop(sender_id, None) + + for receiver_id in self._by_sender.pop(sender_id, ()): + self._by_receiver[receiver_id].discard(sender_id) + + return cleanup + + def _cleanup_bookkeeping(self) -> None: + """Prune unused sender/receiver bookkeeping. Not threadsafe. + + Connecting & disconnecting leaves behind a small amount of bookkeeping + data. Typical workloads using Blinker, for example in most web apps, + Flask, CLI scripts, etc., are not adversely affected by this + bookkeeping. + + With a long-running process performing dynamic signal routing with high + volume, e.g. connecting to function closures, senders are all unique + object instances. Doing all of this over and over may cause memory usage + to grow due to extraneous bookkeeping. (An empty ``set`` for each stale + sender/receiver pair.) + + This method will prune that bookkeeping away, with the caveat that such + pruning is not threadsafe. The risk is that cleanup of a fully + disconnected receiver/sender pair occurs while another thread is + connecting that same pair. If you are in the highly dynamic, unique + receiver/sender situation that has lead you to this method, that failure + mode is perhaps not a big deal for you. + """ + for mapping in (self._by_sender, self._by_receiver): + for ident, bucket in list(mapping.items()): + if not bucket: + mapping.pop(ident, None) + + def _clear_state(self) -> None: + """Disconnect all receivers and senders. Useful for tests.""" + self._weak_senders.clear() + self.receivers.clear() + self._by_sender.clear() + self._by_receiver.clear() + + +class NamedSignal(Signal): + """A named generic notification emitter. The name is not used by the signal + itself, but matches the key in the :class:`Namespace` that it belongs to. + + :param name: The name of the signal within the namespace. + :param doc: The docstring for the signal. + """ + + def __init__(self, name: str, doc: str | None = None) -> None: + super().__init__(doc) + + #: The name of this signal. + self.name: str = name + + def __repr__(self) -> str: + base = super().__repr__() + return f"{base[:-1]}; {self.name!r}>" # noqa: E702 + + +class Namespace(dict[str, NamedSignal]): + """A dict mapping names to signals.""" + + def signal(self, name: str, doc: str | None = None) -> NamedSignal: + """Return the :class:`NamedSignal` for the given ``name``, creating it + if required. Repeated calls with the same name return the same signal. + + :param name: The name of the signal. + :param doc: The docstring of the signal. + """ + if name not in self: + self[name] = NamedSignal(name, doc) + + return self[name] + + +class _PNamespaceSignal(t.Protocol): + def __call__(self, name: str, doc: str | None = None) -> NamedSignal: ... + + +default_namespace: Namespace = Namespace() +"""A default :class:`Namespace` for creating named signals. :func:`signal` +creates a :class:`NamedSignal` in this namespace. +""" + +signal: _PNamespaceSignal = default_namespace.signal +"""Return a :class:`NamedSignal` in :data:`default_namespace` with the given +``name``, creating it if required. Repeated calls with the same name return the +same signal. +""" diff --git a/lib/python3.12/site-packages/blinker/py.typed b/lib/python3.12/site-packages/blinker/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/einops-0.8.2.dist-info/INSTALLER b/lib/python3.12/site-packages/einops-0.8.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/einops-0.8.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/einops-0.8.2.dist-info/METADATA b/lib/python3.12/site-packages/einops-0.8.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..cb156e638fb958fedd1323631c6d5dddc0936b32 --- /dev/null +++ b/lib/python3.12/site-packages/einops-0.8.2.dist-info/METADATA @@ -0,0 +1,394 @@ +Metadata-Version: 2.4 +Name: einops +Version: 0.8.2 +Summary: A new flavour of deep learning operations +Project-URL: Homepage, https://github.com/arogozhnikov/einops +Author: Alex Rogozhnikov +License: MIT +License-File: LICENSE +Keywords: deep learning,einops,machine learning,neural networks,scientific computations,tensor manipulation +Classifier: Intended Audience :: Science/Research +Classifier: License :: OSI Approved :: MIT License +Classifier: Programming Language :: Python :: 3 +Requires-Python: >=3.9 +Description-Content-Type: text/markdown + + + + + + +https://user-images.githubusercontent.com/6318811/177030658-66f0eb5d-e136-44d8-99c9-86ae298ead5b.mp4 + + + + +# einops +[![Run tests](https://github.com/arogozhnikov/einops/actions/workflows/run_tests.yml/badge.svg)](https://github.com/arogozhnikov/einops/actions/workflows/run_tests.yml) +[![PyPI version](https://badge.fury.io/py/einops.svg)](https://badge.fury.io/py/einops) +[![Documentation](https://img.shields.io/badge/documentation-link-blue.svg)](https://einops.rocks/) +![Supported python versions](https://raw.githubusercontent.com/arogozhnikov/einops/main/docs/resources/python_badge.svg) + + +Flexible and powerful tensor operations for readable and reliable code.
+Supports numpy, pytorch, tensorflow, jax, and [others](#supported-frameworks). + +## Recent updates: + +- 0.8.0: tinygrad backend added, small fixes +- 0.7.0: no-hassle `torch.compile`, support of [array api standard](https://data-apis.org/array-api/latest/API_specification/index.html) and more +- 10'000🎉: github reports that more than 10k project use einops +- einops 0.6.1: paddle backend added +- einops 0.6 introduces [packing and unpacking](https://github.com/arogozhnikov/einops/blob/main/docs/4-pack-and-unpack.ipynb) +- einops 0.5: einsum is now a part of einops +- [Einops paper](https://openreview.net/pdf?id=oapKSVM2bcj) is accepted for oral presentation at ICLR 2022 (yes, it worth reading). + Talk recordings are [available](https://iclr.cc/virtual/2022/oral/6603) + + +
+Previous updates +- flax and oneflow backend added +- torch.jit.script is supported for pytorch layers +- powerful EinMix added to einops. [Einmix tutorial notebook](https://github.com/arogozhnikov/einops/blob/main/docs/3-einmix-layer.ipynb) +
+ + + + +## Tweets + +> In case you need convincing arguments for setting aside time to learn about einsum and einops... +[Tim Rocktäschel](https://twitter.com/_rockt/status/1230818967205425152) + +> Writing better code with PyTorch and einops 👌 +[Andrej Karpathy](https://twitter.com/karpathy/status/1290826075916779520) + +> Slowly but surely, einops is seeping in to every nook and cranny of my code. If you find yourself shuffling around bazillion dimensional tensors, this might change your life +[Nasim Rahaman](https://twitter.com/nasim_rahaman/status/1216022614755463169) + +[More testimonials](https://einops.rocks/pages/testimonials/) + + +## Contents + +- [Installation](#Installation) +- [Documentation](https://einops.rocks/) +- [Tutorial](#Tutorials) +- [API micro-reference](#API) +- [Why use einops](#Why-use-einops-notation) +- [Supported frameworks](#Supported-frameworks) +- [Citing](#Citing) +- [Repository](https://github.com/arogozhnikov/einops) and [discussions](https://github.com/arogozhnikov/einops/discussions) + +## Installation + +Plain and simple: +```bash +pip install einops +``` + +(`uv pip install einops` works as well) + +## Tutorials + +Tutorials are the most convenient way to see `einops` in action + +- part 1: [einops fundamentals](https://github.com/arogozhnikov/einops/blob/main/docs/1-einops-basics.ipynb) +- part 2: [einops for deep learning](https://github.com/arogozhnikov/einops/blob/main/docs/2-einops-for-deep-learning.ipynb) +- part 3: [packing and unpacking](https://github.com/arogozhnikov/einops/blob/main/docs/4-pack-and-unpack.ipynb) +- part 4: [improve pytorch code with einops](http://einops.rocks/pytorch-examples.html) + +Kapil Sachdeva recorded a small [intro to einops](https://www.youtube.com/watch?v=xGy75Pjsqzo). + +## API + +`einops` has a minimalistic yet powerful API. + +Three core operations provided ([einops tutorial](https://github.com/arogozhnikov/einops/blob/main/docs/) +shows those cover stacking, reshape, transposition, squeeze/unsqueeze, repeat, tile, concatenate, view and numerous reductions) + +```python +from einops import rearrange, reduce, repeat +# rearrange elements according to the pattern +output_tensor = rearrange(input_tensor, 't b c -> b c t') +# combine rearrangement and reduction +output_tensor = reduce(input_tensor, 'b c (h h2) (w w2) -> b h w c', 'mean', h2=2, w2=2) +# copy along a new axis +output_tensor = repeat(input_tensor, 'h w -> h w c', c=3) +``` + +Later additions to the family are `pack` and `unpack` functions (better than stack/split/concatenate): + +```python +from einops import pack, unpack +# pack and unpack allow reversibly 'packing' multiple tensors into one. +# Packed tensors may be of different dimensionality: +packed, ps = pack([class_token_bc, image_tokens_bhwc, text_tokens_btc], 'b * c') +class_emb_bc, image_emb_bhwc, text_emb_btc = unpack(transformer(packed), ps, 'b * c') +``` + +Finally, einops provides einsum with a support of multi-lettered names: + +```python +from einops import einsum, pack, unpack +# einsum is like ... einsum, generic and flexible dot-product +# but 1) axes can be multi-lettered 2) pattern goes last 3) works with multiple frameworks +C = einsum(A, B, 'b t1 head c, b t2 head c -> b head t1 t2') +``` + +### EinMix + +`EinMix` is a generic linear layer, perfect for MLP Mixers and similar architectures. + +### Layers + +Einops provides layers (`einops` keeps a separate version for each framework) that reflect corresponding functions + +```python +from einops.layers.torch import Rearrange, Reduce +from einops.layers.tensorflow import Rearrange, Reduce +from einops.layers.flax import Rearrange, Reduce +from einops.layers.paddle import Rearrange, Reduce +``` + +
+Example of using layers within a pytorch model +Example given for pytorch, but code in other frameworks is almost identical + +```python +from torch.nn import Sequential, Conv2d, MaxPool2d, Linear, ReLU +from einops.layers.torch import Rearrange + +model = Sequential( + ..., + Conv2d(6, 16, kernel_size=5), + MaxPool2d(kernel_size=2), + # flattening without need to write forward + Rearrange('b c h w -> b (c h w)'), + Linear(16*5*5, 120), + ReLU(), + Linear(120, 10), +) +``` + +No more flatten needed! + +Additionally, torch layers as those are script-able and compile-able. +Operations [are torch.compile-able](https://github.com/arogozhnikov/einops/wiki/Using-torch.compile-with-einops), + but not script-able due to limitations of torch.jit.script. +
+ + + + +## Naming + +`einops` stands for Einstein-Inspired Notation for operations +(though "Einstein operations" is more attractive and easier to remember). + +Notation was loosely inspired by Einstein summation (in particular by `numpy.einsum` operation). + +## Why use `einops` notation?! + + +### Semantic information (being verbose in expectations) + +```python +y = x.view(x.shape[0], -1) +y = rearrange(x, 'b c h w -> b (c h w)') +``` +While these two lines are doing the same job in *some* context, +the second one provides information about the input and output. +In other words, `einops` focuses on interface: *what is the input and output*, not *how* the output is computed. + +The next operation looks similar: + +```python +y = rearrange(x, 'time c h w -> time (c h w)') +``` +but it gives the reader a hint: +this is not an independent batch of images we are processing, +but rather a sequence (video). + +Semantic information makes the code easier to read and maintain. + +### Convenient checks + +Reconsider the same example: + +```python +y = x.view(x.shape[0], -1) # x: (batch, 256, 19, 19) +y = rearrange(x, 'b c h w -> b (c h w)') +``` +The second line checks that the input has four dimensions, +but you can also specify particular dimensions. +That's opposed to just writing comments about shapes since comments don't prevent mistakes, +not tested, and without code review tend to be outdated +```python +y = x.view(x.shape[0], -1) # x: (batch, 256, 19, 19) +y = rearrange(x, 'b c h w -> b (c h w)', c=256, h=19, w=19) +``` + +### Result is strictly determined + +Below we have at least two ways to define the depth-to-space operation +```python +# depth-to-space +rearrange(x, 'b c (h h2) (w w2) -> b (c h2 w2) h w', h2=2, w2=2) +rearrange(x, 'b c (h h2) (w w2) -> b (h2 w2 c) h w', h2=2, w2=2) +``` +There are at least four more ways to do it. Which one is used by the framework? + +These details are ignored, since *usually* it makes no difference, +but it can make a big difference (e.g. if you use grouped convolutions in the next stage), +and you'd like to specify this in your code. + + +### Uniformity + +```python +reduce(x, 'b c (x dx) -> b c x', 'max', dx=2) +reduce(x, 'b c (x dx) (y dy) -> b c x y', 'max', dx=2, dy=3) +reduce(x, 'b c (x dx) (y dy) (z dz) -> b c x y z', 'max', dx=2, dy=3, dz=4) +``` +These examples demonstrated that we don't use separate operations for 1d/2d/3d pooling, +those are all defined in a uniform way. + +Space-to-depth and depth-to space are defined in many frameworks but how about width-to-height? Here you go: + +```python +rearrange(x, 'b c h (w w2) -> b c (h w2) w', w2=2) +``` + + +### Framework independent behavior + +Even simple functions are defined differently by different frameworks + +```python +y = x.flatten() # or flatten(x) +``` + +Suppose `x`'s shape was `(3, 4, 5)`, then `y` has shape ... + +- numpy, pytorch, cupy, chainer, jax: `(60,)` +- keras, tensorflow.layers, gluon: `(3, 20)` + +`einops` works the same way in all frameworks. + + +### Independence of framework terminology + +Example: `tile` vs `repeat` causes lots of confusion. To copy image along width: +```python +np.tile(image, (1, 2)) # in numpy +image.repeat(1, 2) # pytorch's repeat ~ numpy's tile +``` + +With einops you don't need to decipher which axis was repeated: +```python +repeat(image, 'h w -> h (tile w)', tile=2) # in numpy +repeat(image, 'h w -> h (tile w)', tile=2) # in pytorch +repeat(image, 'h w -> h (tile w)', tile=2) # in tf +repeat(image, 'h w -> h (tile w)', tile=2) # in jax +repeat(image, 'h w -> h (tile w)', tile=2) # in cupy +... (etc.) +``` + +[Testimonials](https://einops.rocks/pages/testimonials/) provide users' perspective on the same question. + + +## Supported frameworks + +Einops works with ... + +- [numpy](http://www.numpy.org/) +- [pytorch](https://pytorch.org/) +- [tensorflow](https://www.tensorflow.org/) +- [jax](https://github.com/google/jax) +- [cupy](https://github.com/cupy/cupy) +- [flax](https://github.com/google/flax) (community) +- [paddle](https://github.com/PaddlePaddle/Paddle) (community) +- [oneflow](https://github.com/Oneflow-Inc/oneflow) (community) +- [tinygrad](https://github.com/tinygrad/tinygrad) (community) +- [pytensor](https://github.com/pymc-devs/pytensor) (community) + + + +```python +from einops import rearrange +=> from einops.array_api import rearrange +``` + +But actually it is even better: einops can be used with *any* framework that supports +[Python array API standard](https://data-apis.org/array-api/latest/API_specification/index.html), +to name a few: + +- numpy >= 2.0 +- [MLX](https://github.com/ml-explore/mlx) # yes, einops works with apple's framework +- [pydata/sparse](https://github.com/pydata/sparse) >= 0.15 # and works with sparse tensors +- [cubed](https://github.com/cubed-dev/cubed) # and with distributed tensors too +- [quantco/ndonnx](https://github.com/Quantco/ndonnx) +- jax +- cupy +- dask is supported via [array-api-compat](https://github.com/data-apis/array-api-compat) + + +## Development + +Devcontainer is provided, this environment can be used locally, or on your server, +or within github codespaces. +To start with devcontainers in vs code, clone repo, and click 'Reopen in Devcontainer'. + +Starting from einops 0.8.1, einops distributes tests as a part of package. + +```bash +# pip install einops pytest +python -m einops.tests.run_tests numpy pytorch jax --pip-install +``` + +`numpy pytorch jax` is an _example_, any subset of testable frameworks can be provided. +Every framework is tested against numpy, so it is a requirement for tests. + +Specifying `--pip-install` will install requirements in current virtualenv, +and should be omitted if dependencies are installed locally. + +To build/test docs: + +```bash +hatch run docs:serve # Serving on http://localhost:8000/ +``` + + +## Citing einops + +Please use the following bibtex record + +```text +@inproceedings{ + rogozhnikov2022einops, + title={Einops: Clear and Reliable Tensor Manipulations with Einstein-like Notation}, + author={Alex Rogozhnikov}, + booktitle={International Conference on Learning Representations}, + year={2022}, + url={https://openreview.net/forum?id=oapKSVM2bcj} +} +``` + + +## Supported python versions + +`einops` works with python 3.9 or later. diff --git a/lib/python3.12/site-packages/einops-0.8.2.dist-info/RECORD b/lib/python3.12/site-packages/einops-0.8.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..de4b241cfd8cdc2937a07f2cafd04a29abc029db --- /dev/null +++ b/lib/python3.12/site-packages/einops-0.8.2.dist-info/RECORD @@ -0,0 +1,58 @@ +einops-0.8.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +einops-0.8.2.dist-info/METADATA,sha256=C9ZdsENoikT0opYIwculIcky9_928ji72GlEHhUBJlM,13724 +einops-0.8.2.dist-info/RECORD,, +einops-0.8.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 +einops-0.8.2.dist-info/licenses/LICENSE,sha256=MNmENkKW9R_67K1LAe4SfpUlDFBokY1LZvyWIGcj5DQ,1073 +einops/__init__.py,sha256=MwPcYWGHKsJE0Uxlhf7YMv_Sf6vu9W9Y98hy1mRM6zk,438 +einops/__pycache__/__init__.cpython-312.pyc,, +einops/__pycache__/_backends.cpython-312.pyc,, +einops/__pycache__/_torch_specific.cpython-312.pyc,, +einops/__pycache__/array_api.cpython-312.pyc,, +einops/__pycache__/einops.cpython-312.pyc,, +einops/__pycache__/packing.cpython-312.pyc,, +einops/__pycache__/parsing.cpython-312.pyc,, +einops/_backends.py,sha256=v4snGd66fVWeK0M8e3lNJ9j7_qmd8RMRyJVUTQB5slc,22528 +einops/_torch_specific.py,sha256=QXgLFLDIgDa_Iqtv5gcucEeD3LHALYHJhaXRKgC5f4A,4329 +einops/array_api.py,sha256=7mQ-4yhlWtqmtvcTZyI-pmDOSDbk3qpSCExl4-Vm1-o,5236 +einops/einops.py,sha256=vmx5IWde_ePuzrPDLZZVw_-0Qsare1k8YRRaKt5ZL-8,38079 +einops/experimental/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +einops/experimental/__pycache__/__init__.cpython-312.pyc,, +einops/experimental/__pycache__/indexing.cpython-312.pyc,, +einops/experimental/indexing.py,sha256=yFFflW3-kV6_5PPJU7_jOJsJBCWCWlE4dGlu9gwSPXo,121 +einops/layers/__init__.py,sha256=jijwEY07Yr3rBx8N5zwZ-5qfbl-KX2BcRDK8NJFDaWA,3705 +einops/layers/__pycache__/__init__.cpython-312.pyc,, +einops/layers/__pycache__/_einmix.cpython-312.pyc,, +einops/layers/__pycache__/flax.cpython-312.pyc,, +einops/layers/__pycache__/keras.cpython-312.pyc,, +einops/layers/__pycache__/oneflow.cpython-312.pyc,, +einops/layers/__pycache__/paddle.cpython-312.pyc,, +einops/layers/__pycache__/tensorflow.cpython-312.pyc,, +einops/layers/__pycache__/torch.cpython-312.pyc,, +einops/layers/_einmix.py,sha256=MQr6rymfMw-nzFmhvaLCcee0o7Ai8a1lmluxndWDAjU,11075 +einops/layers/flax.py,sha256=Hk7L31wR2BHL63950-0hy3ivw4rc8NkgGW9jn1AKs-8,2518 +einops/layers/keras.py,sha256=jxD9ZgYLWsKMkCIwojbBNSrY6c1pfXYhE2AMZa0pNXA,217 +einops/layers/oneflow.py,sha256=djcKEZRJrWvxTuSsO4jJVZojieKiDPdUpP4SOpg_zTM,1864 +einops/layers/paddle.py,sha256=YCHffUB4ha7wG-xEOeR9XcTPl48Kr57k_nBEx-J6FVA,1907 +einops/layers/tensorflow.py,sha256=MjGApMbkb_gwAdGya7uGfPZ7olyETsRzbwymfJHii9E,3322 +einops/layers/torch.py,sha256=BV-PzjoijGU9RGea1HCFuIMx4yccicz767M0jJDIH6U,2405 +einops/packing.py,sha256=EZeGLgMRD78hB-0e80ubXvWHukh0_O_wJCf0Hbbdylo,7660 +einops/parsing.py,sha256=vNPYcAs54-KLX1RtbkypGpsSpujzVkj_dQkZlUinLYw,6805 +einops/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +einops/tests/__init__.py,sha256=CpSdrOGBSspTwuG6cAsYMGAwkTq0VuIBrehM7K-SYwU,3531 +einops/tests/__pycache__/__init__.cpython-312.pyc,, +einops/tests/__pycache__/run_tests.cpython-312.pyc,, +einops/tests/__pycache__/test_einsum.cpython-312.pyc,, +einops/tests/__pycache__/test_examples.cpython-312.pyc,, +einops/tests/__pycache__/test_layers.cpython-312.pyc,, +einops/tests/__pycache__/test_ops.cpython-312.pyc,, +einops/tests/__pycache__/test_other.cpython-312.pyc,, +einops/tests/__pycache__/test_packing.cpython-312.pyc,, +einops/tests/__pycache__/test_parsing.cpython-312.pyc,, +einops/tests/run_tests.py,sha256=efJ2apHPOw1BKNIv5qZSNjHpgjSv0rJ4v9hbPbE7yNE,3072 +einops/tests/test_einsum.py,sha256=Nycwg4a-4_Vz7l5f7mj2KIQ7QVs8P2SQIy1rnDcMOG8,11022 +einops/tests/test_examples.py,sha256=6VVkHgov6hpEyoUGbMzFvRfoWTdP1Wr8RPji7G4pPlw,11510 +einops/tests/test_layers.py,sha256=m8RqmBojvVMgr_ET6mhcGpbfYPasg60jUYux7ZlXhSM,18667 +einops/tests/test_ops.py,sha256=4nxEGJWrmEG6cctYHGYuVVdEs5nUN9ziH7ukQYar44M,26884 +einops/tests/test_other.py,sha256=vkWSJr7sn-COewksivTPaLe-GAmzLt7IEZF9TMnt360,13521 +einops/tests/test_packing.py,sha256=AIBNiy8RftTj8dLv1AdNijkTXfrjCOTSw43fuD6Pfjc,10408 +einops/tests/test_parsing.py,sha256=-YAJeJlhDb3U6xYIDRrtm6p4MSNWogeH6l3-np1wUCA,4389 diff --git a/lib/python3.12/site-packages/einops-0.8.2.dist-info/WHEEL b/lib/python3.12/site-packages/einops-0.8.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..12228d414b6cfed7c39d3781c85c63256a1d7fb5 --- /dev/null +++ b/lib/python3.12/site-packages/einops-0.8.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.27.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/python3.12/site-packages/einops-0.8.2.dist-info/licenses/LICENSE b/lib/python3.12/site-packages/einops-0.8.2.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..3a654e906619009358eb2cfe80609bd12b43fa7f --- /dev/null +++ b/lib/python3.12/site-packages/einops-0.8.2.dist-info/licenses/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Alex Rogozhnikov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/lib/python3.12/site-packages/functorch/__init__.py b/lib/python3.12/site-packages/functorch/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0aef38c8a9bb84a9833c4c2c9c34ad528d564b32 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/__init__.py @@ -0,0 +1,39 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import torch +from torch._functorch.deprecated import ( + combine_state_for_ensemble, + functionalize, + grad, + grad_and_value, + hessian, + jacfwd, + jacrev, + jvp, + make_functional, + make_functional_with_buffers, + vjp, + vmap, +) + +# utilities. Maybe these should go in their own namespace in the future? +from torch._functorch.make_functional import ( + FunctionalModule, + FunctionalModuleWithBuffers, +) + +# Was never documented +from torch._functorch.python_key import make_fx + + +# Top-level APIs. Please think carefully before adding something to the +# top-level namespace: +# - private helper functions should go into torch._functorch +# - very experimental things should go into functorch.experimental +# - compilation related things should go into functorch.compile + + +__version__ = torch.__version__ diff --git a/lib/python3.12/site-packages/functorch/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..048b58d2e3281c27fe934ba50620e0a6e9688ec6 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/_src/__init__.py b/lib/python3.12/site-packages/functorch/_src/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/functorch/_src/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/_src/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe7718b00e9b5a2594a2a22a03f4ad4fd9cb8b7f Binary files /dev/null and b/lib/python3.12/site-packages/functorch/_src/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/_src/aot_autograd/__init__.py b/lib/python3.12/site-packages/functorch/_src/aot_autograd/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..94f258df84ba8730208768fc44222bee4b3ebc33 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/_src/aot_autograd/__init__.py @@ -0,0 +1,8 @@ +# This file has moved to under torch/_functorch. It is not public API. +# If you are not a PyTorch developer and you are relying on the following +# imports, please file an issue. +from torch._functorch.aot_autograd import ( + aot_autograd_decompositions, + KNOWN_TYPES, + PytreeThunk, +) diff --git a/lib/python3.12/site-packages/functorch/_src/aot_autograd/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/_src/aot_autograd/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c10936469b4aefac1c584b5099460689890c948e Binary files /dev/null and b/lib/python3.12/site-packages/functorch/_src/aot_autograd/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/_src/eager_transforms/__init__.py b/lib/python3.12/site-packages/functorch/_src/eager_transforms/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6052b5548f4af3dbc6d9d45b0ffe72a8d5013d41 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/_src/eager_transforms/__init__.py @@ -0,0 +1,7 @@ +# This file has moved to under torch/_functorch. It is not public API. +# If you are not a PyTorch developer and you are relying on the following +# imports, please file an issue. +from torch._functorch.eager_transforms import ( + _assert_wrapped_functional, + _unwrap_functional_tensor, +) diff --git a/lib/python3.12/site-packages/functorch/_src/eager_transforms/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/_src/eager_transforms/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..76ee3be1938336bd46df845a3696f5d6d83f198c Binary files /dev/null and b/lib/python3.12/site-packages/functorch/_src/eager_transforms/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/_src/make_functional/__init__.py b/lib/python3.12/site-packages/functorch/_src/make_functional/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3de7787df0c3304207b42b51e9fb62da9d33c7d0 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/_src/make_functional/__init__.py @@ -0,0 +1,4 @@ +# This file has moved to under torch/_functorch. It is not public API. +# If you are not a PyTorch developer and you are relying on the following +# imports, please file an issue. +from torch._functorch.make_functional import _swap_state diff --git a/lib/python3.12/site-packages/functorch/_src/make_functional/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/_src/make_functional/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bab24eff1210f946262b002678834c17d19d4247 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/_src/make_functional/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/_src/vmap/__init__.py b/lib/python3.12/site-packages/functorch/_src/vmap/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..dc90517753e50f92362ba954248e31f69f7cfcd5 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/_src/vmap/__init__.py @@ -0,0 +1,16 @@ +# This file has moved to under torch/_functorch. It is not public API. +# If you are not a PyTorch developer and you are relying on the following +# imports, please file an issue. +from torch._functorch.vmap import ( + _add_batch_dim, + _broadcast_to_and_flatten, + _create_batched_inputs, + _get_name, + _process_batched_inputs, + _remove_batch_dim, + _unwrap_batched, + _validate_and_get_batch_size, + Tensor, + tree_flatten, + tree_unflatten, +) diff --git a/lib/python3.12/site-packages/functorch/_src/vmap/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/_src/vmap/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5ed8881e045f2be91380a015f572a34f468d7f0 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/_src/vmap/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/compile/__init__.py b/lib/python3.12/site-packages/functorch/compile/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e7548a5ff6b91bae4fa561f0de7ad5d3492eda05 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/compile/__init__.py @@ -0,0 +1,30 @@ +from torch._functorch import config +from torch._functorch.aot_autograd import ( + aot_function, + aot_module, + aot_module_simplified, + compiled_function, + compiled_module, + get_aot_compilation_context, + get_aot_graph_name, + get_graph_being_compiled, + make_boxed_compiler, + make_boxed_func, +) +from torch._functorch.compilers import ( + debug_compile, + default_decompositions, + draw_graph_compile, + memory_efficient_fusion, + nnc_jit, + nop, + print_compile, + ts_compile, +) +from torch._functorch.fx_minifier import minifier +from torch._functorch.partitioners import ( + default_partition, + draw_graph, + min_cut_rematerialization_partition, +) +from torch._functorch.python_key import pythonkey_decompose diff --git a/lib/python3.12/site-packages/functorch/compile/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/compile/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..659664ffbfbf9decb452d89a484245889ba699e6 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/compile/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__init__.py b/lib/python3.12/site-packages/functorch/dim/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..a6de6ad59e9501b13cf5ba934c810e53ad61f6a3 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/__init__.py @@ -0,0 +1,181 @@ +import dis +import inspect +from typing import Sequence, Union + +import functorch._C +import torch +from functorch._C import dim as _C + +from .tree_map import tree_flatten, tree_map +from .wrap_type import wrap_type + + +_C._patch_tensor_class() +dims, DimList, dimlists = _C.dims, _C.DimList, _C.dimlists + + +class DimensionMismatchError(Exception): + pass + + +class DimensionBindError(Exception): + pass + + +from . import op_properties + + +# use dict to avoid writing C++ bindings for set +pointwise = dict.fromkeys(op_properties.pointwise, True) + +use_c = True +if not use_c: + from . import reference + + +class _Tensor: + # fast path around slow wrapping/unwrapping logic for simply queries used + # by the implementation... + + @property + def dims(self): + return tuple(d for d in self._levels if isinstance(d, Dim)) + + def dim(self): + return self.ndim + + if use_c: + __torch_function__ = classmethod(_C.__torch_function__) + expand = _C._instancemethod(_C.expand) + else: + __torch_function__ = reference.__torch_function__ + expand = reference.expand + + index = _C._instancemethod(_C.index) + + def __repr__(self): + tensor, levels, ndim = self._tensor, self._levels, self.ndim + return f"{tensor}\nwith dims={tuple(l + ndim if isinstance(l, int) else l for l in levels)} sizes={tuple(tensor.size())}" + + +TensorLike = (_Tensor, torch.Tensor) + + +class Dim(_C.Dim, _Tensor): + # note that _C.Dim comes before tensor because we want the Dim API for things like size to take precendence. + # Tensor defines format, but we want to print Dims with special formatting + __format__ = object.__format__ + + +class Tensor(_Tensor, _C.Tensor): + if not use_c: + from_batched = staticmethod(_C.Tensor_from_batched) + from_positional = staticmethod(_C.Tensor_from_positional) + sum = _C._instancemethod(_C.Tensor_sum) + + +def cat(tensors, dim, new_dim): + n = dims() + return stack(tensors, n, dim).index([n, dim], new_dim) + + +if use_c: + _wrap = _C._wrap + + def _def(name, *args, **kwargs): + orig = getattr(torch.Tensor, name) + setattr(_Tensor, name, _C._instancemethod(_wrap(orig, *args, **kwargs))) + + t__getitem__ = _C._instancemethod(_C.__getitem__) + stack = _C.stack + split = _C._instancemethod(_C.split) +else: + _wrap, _def = reference._wrap, reference._def + t__getitem__ = reference.t__getitem__ + stack = reference.stack + split = reference.split + +# note: there is no python reference +t__setitem__ = _C._instancemethod(_C.__setitem__) +# this is patched in the C API because otherwise torch.Tensor will +# no longer be considered a sequence and things will break +# torch.Tensor.__getitem__ = t__getitem__ + +_Tensor.__getitem__ = t__getitem__ +# torch.Tensor.__setitem__ = t__setitem__ +_Tensor.__setitem__ = t__setitem__ + +torch.Tensor.split = split +_Tensor.split = split +torch.Tensor.expand = _C._instancemethod(_C.expand) +torch.Tensor.index = _C._instancemethod(_C.index) +wrap_type(use_c, _Tensor, torch.Tensor, _Tensor.__torch_function__) +del _Tensor.ndim + +if use_c: + _Tensor.order = _C._instancemethod(_C.order) +else: + _Tensor.order = reference.positional + +_def("mean") +_def("sum") +_def("all") +_def("amax") +_def("amin") +_def("aminmax") +_def("any") +_def("count_nonzero") +_def("logsumexp") +_def("nanmean") +_def("nansum") +_def("prod") +_def("std", keepdim_offset=2) +_def("var", keepdim_offset=2) +_def("max", single_dim=True) +_def("min", single_dim=True) +_def("argmax", single_dim=True) +_def("argmin", single_dim=True) +_def("kthvalue", single_dim=True) +_def("median", single_dim=True) +_def("nanmedian", single_dim=True) +_def("mode", single_dim=True) +_def("sort", reduce=False) +_def("argsort", reduce=False) +_def("unbind", single_dim=True) +_def("chunk", dim_offset=1, reduce=False) +_def("cummax", single_dim=True, reduce=False) +_def("cummin", single_dim=True, reduce=False) +_def("cumprod", single_dim=True, reduce=False) +_def("cumprod_", single_dim=True, reduce=False) +_def("cumsum", single_dim=True, reduce=False) +_def("cumsum_", single_dim=True, reduce=False) +_def("logcumsumexp", single_dim=True, reduce=False) +_def("renorm", dim_offset=1, single_dim=True, reduce=False) +_def("softmax", single_dim=True, reduce=False) +softmax = _wrap(torch.nn.functional.softmax, single_dim=True, reduce=False) + +# stuff to handle in the future, because they require special +# binding logic for dims +# cross +# diag_embed +# diagonal +# diagonal_scatter +# diff +# nanquantile +# quantile +# roll +# rot90 +# topk (new dimes on output) +# should these all be subsumed by inplace indexing? +# index_add_ +# index_add +# index_copy +# index_copy_ +# index_fill +# index_fill_ +# index_select +# scatter +# scatter_ +# scatter_add +# scatter_add_ +# scatter_reduce diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81ec64f61afcd7c1206128496b543ec56dd64f14 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/batch_tensor.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/batch_tensor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb7a6edbff3ec386db0b7bb3d36da263d640af42 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/batch_tensor.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/delayed_mul_tensor.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/delayed_mul_tensor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0a13db6a8e973b06deaa4b2d5ff48621fc1b9582 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/delayed_mul_tensor.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/dim.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/dim.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f79d8f5540a540571621d5bee076e6359fb57af9 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/dim.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/magic_trace.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/magic_trace.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5126818d27401c7481bd043acbe556de3a3c5e3e Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/magic_trace.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/op_properties.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/op_properties.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d88aa30d5e01dc6eb43cd6917ed3c9754f8a8359 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/op_properties.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/reference.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/reference.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1a609435d0856f2c3b3f75fb29a370bd7923bbf3 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/reference.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/tree_map.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/tree_map.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3940e752bee927fbe3e058d331c7a8e50a1f761a Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/tree_map.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/__pycache__/wrap_type.cpython-312.pyc b/lib/python3.12/site-packages/functorch/dim/__pycache__/wrap_type.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd06c024a17955846a1207fa5deeded87b7bb23f Binary files /dev/null and b/lib/python3.12/site-packages/functorch/dim/__pycache__/wrap_type.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/dim/batch_tensor.py b/lib/python3.12/site-packages/functorch/dim/batch_tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..dae9b270896e988151741409b666a82ba1aba5a4 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/batch_tensor.py @@ -0,0 +1,26 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +from contextlib import contextmanager + +from torch._C._functorch import _vmap_add_layers, _vmap_remove_layers + + +_enabled = False + + +@contextmanager +def _enable_layers(dims): + global _enabled + assert not _enabled + input = sorted((d._level, d.size) for d in dims if not isinstance(d, int)) + n = len(input) + try: + _vmap_add_layers(input) + _enabled = True + yield + finally: + _enabled = False + _vmap_remove_layers(n) diff --git a/lib/python3.12/site-packages/functorch/dim/delayed_mul_tensor.py b/lib/python3.12/site-packages/functorch/dim/delayed_mul_tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..3c136cfe1247dbd7dcc38777e6326b360619941b --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/delayed_mul_tensor.py @@ -0,0 +1,76 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import torch + +from . import _Tensor, Tensor +from .reference import _dims, _enable_layers, llist, ltuple + + +class DelayedMulTensor(_Tensor): + def __init__(self, lhs, rhs): + self._lhs, self._rhs = lhs, rhs + self._data = None + self._levels_data = None + self._has_device = lhs._has_device or rhs._has_device + self._batchtensor_data = None + self._tensor_data = None + + @property + def _levels(self): + if self._levels_data is None: + levels = llist(self._lhs._levels) + for l in self._rhs._levels: + if l not in levels: + levels.append(l) + self._levels_data = ltuple(levels) + return self._levels_data + + @property + def _batchtensor(self): + if self._batchtensor_data is None: + with _enable_layers(self._levels): + print("bt multiply fallback") + self._batchtensor_data = self._lhs._batchtensor * self._rhs._batchtensor + return self._batchtensor_data + + @property + def _tensor(self): + if self._tensor_data is None: + self._tensor_data = Tensor.from_batched( + self._batchtensor, self._has_device + )._tensor + return self._tensor_data + + @property + def ndim(self): + return self._batchtensor.ndim + + @property + def dims(self): + return ltuple(super().dims) + + def sum(self, dim): + dims = _dims(dim, 0, False, False) + n = ord("a") + all_levels = self._levels + + def to_char(d): + return chr(n + all_levels.index(d)) + + plhs, levelslhs = self._lhs._tensor, self._lhs._levels + prhs, levelsrhs = self._rhs._tensor, self._rhs._levels + new_levels = [l for l in self._levels if l not in dims] + fmt = "".join( + [ + *(to_char(d) for d in levelslhs), + ",", + *(to_char(d) for d in levelsrhs), + "->", + *(to_char(d) for d in new_levels), + ] + ) + result_data = torch.einsum(fmt, (plhs, prhs)) + return Tensor.from_positional(result_data, new_levels, True) diff --git a/lib/python3.12/site-packages/functorch/dim/dim.py b/lib/python3.12/site-packages/functorch/dim/dim.py new file mode 100644 index 0000000000000000000000000000000000000000..9a4b56866484905cc0bd693c865d03c937aa1b75 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/dim.py @@ -0,0 +1,120 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import dis +import inspect +from dataclasses import dataclass +from typing import Union + +from . import DimList + + +_vmap_levels = [] + + +@dataclass +class LevelInfo: + level: int + alive: bool = True + + +class Dim: + def __init__(self, name: str, size: Union[None, int] = None): + self.name = name + self._size = None + self._vmap_level = None + if size is not None: + self.size = size + + def __del__(self): + if self._vmap_level is not None: + _vmap_active_levels[self._vmap_stack].alive = False # noqa: F821 + while ( + not _vmap_levels[-1].alive and current_level() == _vmap_levels[-1].level # noqa: F821 + ): + _vmap_decrement_nesting() # noqa: F821 + _vmap_levels.pop() + + @property + def size(self): + assert self.is_bound + return self._size + + @size.setter + def size(self, size: int): + from . import DimensionBindError + + if self._size is None: + self._size = size + self._vmap_level = _vmap_increment_nesting(size, "same") # noqa: F821 + self._vmap_stack = len(_vmap_levels) + _vmap_levels.append(LevelInfo(self._vmap_level)) + + elif self._size != size: + raise DimensionBindError( + f"Dim '{self}' previously bound to a dimension of size {self._size} cannot bind to a dimension of size {size}" + ) + + @property + def is_bound(self): + return self._size is not None + + def __repr__(self): + return self.name + + +def extract_name(inst): + assert inst.opname == "STORE_FAST" or inst.opname == "STORE_NAME" + return inst.argval + + +_cache = {} + + +def dims(lists=0): + frame = inspect.currentframe() + assert frame is not None + calling_frame = frame.f_back + assert calling_frame is not None + code, lasti = calling_frame.f_code, calling_frame.f_lasti + key = (code, lasti) + if key not in _cache: + first = lasti // 2 + 1 + instructions = list(dis.get_instructions(calling_frame.f_code)) + unpack = instructions[first] + + if unpack.opname == "STORE_FAST" or unpack.opname == "STORE_NAME": + # just a single dim, not a list + name = unpack.argval + ctor = Dim if lists == 0 else DimList + _cache[key] = lambda: ctor(name=name) + else: + assert unpack.opname == "UNPACK_SEQUENCE" + ndims = unpack.argval + names = tuple( + extract_name(instructions[first + 1 + i]) for i in range(ndims) + ) + first_list = len(names) - lists + _cache[key] = lambda: tuple( + Dim(n) if i < first_list else DimList(name=n) + for i, n in enumerate(names) + ) + return _cache[key]() + + +def _dim_set(positional, arg): + def convert(a): + if isinstance(a, Dim): + return a + else: + assert isinstance(a, int) + return positional[a] + + if arg is None: + return positional + elif not isinstance(arg, (Dim, int)): + return tuple(convert(a) for a in arg) + else: + return (convert(arg),) diff --git a/lib/python3.12/site-packages/functorch/dim/magic_trace.py b/lib/python3.12/site-packages/functorch/dim/magic_trace.py new file mode 100644 index 0000000000000000000000000000000000000000..5c962a898ca79cfe3d8af7432aacc3802d4f4ade --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/magic_trace.py @@ -0,0 +1,42 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import os +import signal +import subprocess +from contextlib import contextmanager + + +@contextmanager +def magic_trace(output="trace.fxt", magic_trace_cache="/tmp/magic-trace"): + pid = os.getpid() + if not os.path.exists(magic_trace_cache): + print(f"Downloading magic_trace to: {magic_trace_cache}") + subprocess.run( + [ + "wget", + "-O", + magic_trace_cache, + "-q", + "https://github.com/janestreet/magic-trace/releases/download/v1.0.2/magic-trace", + ] + ) + subprocess.run(["chmod", "+x", magic_trace_cache]) + args = [magic_trace_cache, "attach", "-pid", str(pid), "-o", output] + p = subprocess.Popen(args, stderr=subprocess.PIPE, encoding="utf-8") + while True: + x = p.stderr.readline() + print(x) + if "Attached" in x: + break + try: + yield + finally: + p.send_signal(signal.SIGINT) + r = p.wait() + print(p.stderr.read()) + p.stderr.close() + if r != 0: + raise ValueError(f"magic_trace exited abnormally: {r}") diff --git a/lib/python3.12/site-packages/functorch/dim/op_properties.py b/lib/python3.12/site-packages/functorch/dim/op_properties.py new file mode 100644 index 0000000000000000000000000000000000000000..01313f71f030d58ce76c15c7f8516c4a0bdcf48a --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/op_properties.py @@ -0,0 +1,312 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. +import torch + + +# pointwise operators can go through a faster pathway + +tensor_magic_methods = ["add", ""] +pointwise_magic_methods_with_reverse = ( + "add", + "sub", + "mul", + "floordiv", + "div", + "truediv", + "mod", + "pow", + "lshift", + "rshift", + "and", + "or", + "xor", +) +pointwise_magic_methods = ( + *(x for m in pointwise_magic_methods_with_reverse for x in (m, "r" + m)), + "eq", + "gt", + "le", + "lt", + "ge", + "gt", + "ne", + "neg", + "pos", + "abs", + "invert", + "iadd", + "isub", + "imul", + "ifloordiv", + "idiv", + "itruediv", + "imod", + "ipow", + "ilshift", + "irshift", + "iand", + "ior", + "ixor", + "int", + "long", + "float", + "complex", +) + +pointwise_methods = (*(f"__{m}__" for m in pointwise_magic_methods),) + +pointwise = ( + *(getattr(torch.Tensor, m) for m in pointwise_methods), + torch.nn.functional.dropout, + torch.where, + torch.Tensor.abs, + torch.abs, + torch.Tensor.acos, + torch.acos, + torch.Tensor.acosh, + torch.acosh, + torch.Tensor.add, + torch.add, + torch.Tensor.addcdiv, + torch.addcdiv, + torch.Tensor.addcmul, + torch.addcmul, + torch.Tensor.addr, + torch.addr, + torch.Tensor.angle, + torch.angle, + torch.Tensor.asin, + torch.asin, + torch.Tensor.asinh, + torch.asinh, + torch.Tensor.atan, + torch.atan, + torch.Tensor.atan2, + torch.atan2, + torch.Tensor.atanh, + torch.atanh, + torch.Tensor.bitwise_and, + torch.bitwise_and, + torch.Tensor.bitwise_left_shift, + torch.bitwise_left_shift, + torch.Tensor.bitwise_not, + torch.bitwise_not, + torch.Tensor.bitwise_or, + torch.bitwise_or, + torch.Tensor.bitwise_right_shift, + torch.bitwise_right_shift, + torch.Tensor.bitwise_xor, + torch.bitwise_xor, + torch.Tensor.ceil, + torch.ceil, + torch.celu, + torch.nn.functional.celu, + torch.Tensor.clamp, + torch.clamp, + torch.Tensor.clamp_max, + torch.clamp_max, + torch.Tensor.clamp_min, + torch.clamp_min, + torch.Tensor.copysign, + torch.copysign, + torch.Tensor.cos, + torch.cos, + torch.Tensor.cosh, + torch.cosh, + torch.Tensor.deg2rad, + torch.deg2rad, + torch.Tensor.digamma, + torch.digamma, + torch.Tensor.div, + torch.div, + torch.dropout, + torch.nn.functional.dropout, + torch.nn.functional.elu, + torch.Tensor.eq, + torch.eq, + torch.Tensor.erf, + torch.erf, + torch.Tensor.erfc, + torch.erfc, + torch.Tensor.erfinv, + torch.erfinv, + torch.Tensor.exp, + torch.exp, + torch.Tensor.exp2, + torch.exp2, + torch.Tensor.expm1, + torch.expm1, + torch.feature_dropout, + torch.Tensor.float_power, + torch.float_power, + torch.Tensor.floor, + torch.floor, + torch.Tensor.floor_divide, + torch.floor_divide, + torch.Tensor.fmod, + torch.fmod, + torch.Tensor.frac, + torch.frac, + torch.Tensor.frexp, + torch.frexp, + torch.Tensor.gcd, + torch.gcd, + torch.Tensor.ge, + torch.ge, + torch.nn.functional.gelu, + torch.nn.functional.glu, + torch.Tensor.gt, + torch.gt, + torch.Tensor.hardshrink, + torch.hardshrink, + torch.nn.functional.hardshrink, + torch.nn.functional.hardsigmoid, + torch.nn.functional.hardswish, + torch.nn.functional.hardtanh, + torch.Tensor.heaviside, + torch.heaviside, + torch.Tensor.hypot, + torch.hypot, + torch.Tensor.i0, + torch.i0, + torch.Tensor.igamma, + torch.igamma, + torch.Tensor.igammac, + torch.igammac, + torch.Tensor.isclose, + torch.isclose, + torch.Tensor.isfinite, + torch.isfinite, + torch.Tensor.isinf, + torch.isinf, + torch.Tensor.isnan, + torch.isnan, + torch.Tensor.isneginf, + torch.isneginf, + torch.Tensor.isposinf, + torch.isposinf, + torch.Tensor.isreal, + torch.isreal, + torch.Tensor.kron, + torch.kron, + torch.Tensor.lcm, + torch.lcm, + torch.Tensor.ldexp, + torch.ldexp, + torch.Tensor.le, + torch.le, + torch.nn.functional.leaky_relu, + torch.Tensor.lerp, + torch.lerp, + torch.Tensor.lgamma, + torch.lgamma, + torch.Tensor.log, + torch.log, + torch.Tensor.log10, + torch.log10, + torch.Tensor.log1p, + torch.log1p, + torch.Tensor.log2, + torch.log2, + torch.nn.functional.logsigmoid, + torch.Tensor.logical_and, + torch.logical_and, + torch.Tensor.logical_not, + torch.logical_not, + torch.Tensor.logical_or, + torch.logical_or, + torch.Tensor.logical_xor, + torch.logical_xor, + torch.Tensor.logit, + torch.logit, + torch.Tensor.lt, + torch.lt, + torch.Tensor.maximum, + torch.maximum, + torch.Tensor.minimum, + torch.minimum, + torch.nn.functional.mish, + torch.Tensor.mvlgamma, + torch.mvlgamma, + torch.Tensor.nan_to_num, + torch.nan_to_num, + torch.Tensor.ne, + torch.ne, + torch.Tensor.neg, + torch.neg, + torch.Tensor.nextafter, + torch.nextafter, + torch.Tensor.outer, + torch.outer, + torch.polar, + torch.Tensor.polygamma, + torch.polygamma, + torch.Tensor.positive, + torch.positive, + torch.Tensor.pow, + torch.pow, + torch.Tensor.prelu, + torch.prelu, + torch.nn.functional.prelu, + torch.Tensor.rad2deg, + torch.rad2deg, + torch.Tensor.reciprocal, + torch.reciprocal, + torch.Tensor.relu, + torch.relu, + torch.nn.functional.relu, + torch.nn.functional.relu6, + torch.Tensor.remainder, + torch.remainder, + torch.Tensor.round, + torch.round, + torch.rrelu, + torch.nn.functional.rrelu, + torch.Tensor.rsqrt, + torch.rsqrt, + torch.rsub, + torch.selu, + torch.nn.functional.selu, + torch.Tensor.sgn, + torch.sgn, + torch.Tensor.sigmoid, + torch.sigmoid, + torch.nn.functional.sigmoid, + torch.Tensor.sign, + torch.sign, + torch.Tensor.signbit, + torch.signbit, + torch.nn.functional.silu, + torch.Tensor.sin, + torch.sin, + torch.Tensor.sinc, + torch.sinc, + torch.Tensor.sinh, + torch.sinh, + torch.nn.functional.softplus, + torch.nn.functional.softshrink, + torch.Tensor.sqrt, + torch.sqrt, + torch.Tensor.square, + torch.square, + torch.Tensor.sub, + torch.sub, + torch.Tensor.tan, + torch.tan, + torch.Tensor.tanh, + torch.tanh, + torch.nn.functional.tanh, + torch.threshold, + torch.nn.functional.threshold, + torch.trapz, + torch.Tensor.true_divide, + torch.true_divide, + torch.Tensor.trunc, + torch.trunc, + torch.Tensor.xlogy, + torch.xlogy, + torch.rand_like, +) diff --git a/lib/python3.12/site-packages/functorch/dim/reference.py b/lib/python3.12/site-packages/functorch/dim/reference.py new file mode 100644 index 0000000000000000000000000000000000000000..6453a441b9445b58a02e3f917ff0746790908eae --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/reference.py @@ -0,0 +1,643 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +# reference python implementations for C ops +import torch +from functorch._C import dim as _C + +from . import op_properties +from .batch_tensor import _enable_layers +from .tree_map import tree_flatten, tree_map + + +DimList = _C.DimList +import operator +from functools import reduce + + +# use dict to avoid writing C++ bindings for set +pointwise = set(op_properties.pointwise) + + +def prod(x): + return reduce(operator.mul, x, 1) + + +def _wrap_dim(d, N, keepdim): + from . import Dim + + if isinstance(d, Dim): + assert not keepdim, "cannot preserve first-class dimensions with keepdim=True" + return d + elif d >= 0: + return d - N + else: + return d + + +def _dims(d, N, keepdim, single_dim): + from . import Dim + + if isinstance(d, (Dim, int)): + return ltuple((_wrap_dim(d, N, keepdim),)) + assert not single_dim, f"expected a single dimension or int but found: {d}" + return ltuple(_wrap_dim(x, N, keepdim) for x in d) + + +def _bind_dims_to_size(lhs_size, rhs, lhs_debug): + from . import DimensionMismatchError + + not_bound = tuple((i, r) for i, r in enumerate(rhs) if not r.is_bound) + if len(not_bound) == 1: + idx, d = not_bound[0] + rhs_so_far = prod(r.size for r in rhs if r.is_bound) + if lhs_size % rhs_so_far != 0: + rhs_s = tuple("?" if not r.is_bound else str(r.size) for r in rhs) + raise DimensionMismatchError( + f"inferred dimension does not evenly fit into larger dimension: {lhs_size} vs {rhs_s}" + ) + new_size = lhs_size // rhs_so_far + d.size = new_size + elif len(not_bound) > 1: + rhs_s = tuple("?" if not r.is_bound else str(r.size) for r in rhs) + raise DimensionMismatchError( + f"cannot infer the size of two dimensions at once: {rhs} with sizes {rhs_s}" + ) + else: + rhs_size = prod(r.size for r in rhs) + if lhs_size != rhs_size: + raise DimensionMismatchError( + f"Dimension sizes to do not match ({lhs_size} != {rhs_size}) when matching {lhs_debug} to {rhs}" + ) + + +def _tensor_levels(inp): + from . import _Tensor + + if isinstance(inp, _Tensor): + return inp._tensor, llist(inp._levels), inp._has_device + else: + return inp, llist(range(-inp.ndim, 0)), True + + +def _match_levels(v, from_levels, to_levels): + view = [] + permute = [] + requires_view = False + size = v.size() + for t in to_levels: + try: + idx = from_levels.index(t) + permute.append(idx) + view.append(size[idx]) + except ValueError: + view.append(1) + requires_view = True + if permute != list(range(len(permute))): + v = v.permute(*permute) + if requires_view: + v = v.view(*view) + return v + + +# make a single dimension positional but do not permute it, +# used to do multi-tensor operators where the dim being acted on +# should not physically move if possible +def _positional_no_permute(self, dim, expand_dim=False): + from . import Tensor + + ptensor, levels = self._tensor, llist(self._levels) + try: + idx = levels.index(dim) + except ValueError: + if not expand_dim: + raise + idx = 0 + ptensor = ptensor.expand(dim.size, *ptensor.size()) + levels.insert(0, 0) + idx_batched = 0 + for i in range(idx): + if isinstance(levels[i], int): + levels[i] -= 1 + idx_batched += 1 + levels[idx] = -idx_batched - 1 + return Tensor.from_positional(ptensor, levels, self._has_device), idx_batched + + +def seq(a, b): + from . import Dim + + if isinstance(a, Dim) != isinstance(b, Dim): + return False + if isinstance(a, Dim): + return a is b + else: + return a == b + + +class isin: + def __contains__(self, item): + for x in self: + if seq(item, x): + return True + return False + + def index(self, item): + for i, x in enumerate(self): + if seq(item, x): + return i + raise ValueError + + +class llist(isin, list): + pass + + +class ltuple(isin, tuple): + pass + + +empty_dict = {} + + +@classmethod +def __torch_function__(self, orig, cls, args, kwargs=empty_dict): + from . import _Tensor, Tensor, TensorLike + from .delayed_mul_tensor import DelayedMulTensor + + if orig is torch.Tensor.__mul__: + lhs, rhs = args + if ( + isinstance(lhs, _Tensor) + and isinstance(rhs, _Tensor) + and lhs.ndim == 0 + and rhs.ndim == 0 + ): + return DelayedMulTensor(lhs, rhs) + all_dims = llist() + flat_args, unflatten = tree_flatten((args, kwargs)) + device_holding_tensor = None + for f in flat_args: + if isinstance(f, _Tensor): + if f._has_device: + device_holding_tensor = f._batchtensor + for d in f.dims: + if d not in all_dims: + all_dims.append(d) + + def unwrap(t): + if isinstance(t, _Tensor): + r = t._batchtensor + if device_holding_tensor is not None and not t._has_device: + r = r.to(device=device_holding_tensor.device) + return r + return t + + if orig in pointwise: + result_levels = llist() + to_expand = [] + for i, f in enumerate(flat_args): + if isinstance(f, TensorLike): + ptensor, levels, _ = _tensor_levels(f) + if ( + isinstance(f, _Tensor) + and not f._has_device + and device_holding_tensor is not None + ): + ptensor = ptensor.to(device=device_holding_tensor.device) + flat_args[i] = ptensor + for l in levels: + if l not in result_levels: + result_levels.append(l) + to_expand.append((i, levels)) + + for i, levels in to_expand: + flat_args[i] = _match_levels(flat_args[i], levels, result_levels) + args, kwargs = unflatten(flat_args) + result = orig(*args, **kwargs) + + def wrap(t): + if isinstance(t, TensorLike): + return Tensor.from_positional( + t, result_levels, device_holding_tensor is not None + ) + return t + + return tree_map(wrap, result) + else: + + def wrap(t): + if isinstance(t, TensorLike): + return Tensor.from_batched(t, device_holding_tensor is not None) + return t + + with _enable_layers(all_dims): + print(f"batch_tensor for {orig}") + args, kwargs = unflatten(unwrap(f) for f in flat_args) + result = orig(*args, **kwargs) + # print("END", orig) + return tree_map(wrap, result) + + +def positional(self, *dims): + from . import Dim, DimensionBindError, Tensor + + ptensor, levels = self._tensor, llist(self._levels) + flat_dims = llist() + view = [] + needs_view = False + ndim = self.ndim + for d in dims: + if isinstance(d, DimList): + flat_dims.extend(d) + view.extend(e.size for e in d) + elif isinstance(d, Dim): + flat_dims.append(d) + view.append(d.size) + elif isinstance(d, int): + d = _wrap_dim(d, ndim, False) + flat_dims.append(d) + view.append(ptensor.size(d)) + else: + flat_dims.extend(d) + view.append(prod(e.size for e in d)) + needs_view = True + + permute = list(range(len(levels))) + for i, d in enumerate(flat_dims): + try: + idx = levels.index(d) + except ValueError as e: + raise DimensionBindError( + f"tensor of dimensions {self.dims} does not contain dim {d}" + ) from e + p = permute[idx] + del levels[idx] + del permute[idx] + levels.insert(i, 0) + permute.insert(i, p) + ptensor = ptensor.permute(*permute) + seen = 0 + for i in range(len(levels) - 1, -1, -1): + if isinstance(levels[i], int): + seen += 1 + levels[i] = -seen + result = Tensor.from_positional(ptensor, levels, self._has_device) + if needs_view: + result = result.reshape(*view, *result.size()[len(flat_dims) :]) + return result + + +def _contains_dim(input): + from . import Dim + + for i in input: + if isinstance(i, Dim): + return True + + +def expand(self, *sizes): + if not _contains_dim(sizes): + return self.__torch_function__(torch.Tensor.expand, None, (self, *sizes)) + dims = sizes + sizes = [d.size for d in dims] + [-1] * self.ndim + self = self.expand(*sizes) + return self[dims] + + +_not_present = object() + + +def _getarg(name, offset, args, kwargs, default): + if len(args) > offset: + return args[offset] + return kwargs.get(name, default) + + +def _patcharg(name, offset, args, kwargs, value): + if len(args) > offset: + args[offset] = value + else: + kwargs[name] = value + + +def _wrap( + orig, dim_offset=0, keepdim_offset=1, dim_name="dim", single_dim=False, reduce=True +): + from . import Dim, Tensor, TensorLike + + def fn(self, *args, **kwargs): + dim = _getarg(dim_name, dim_offset, args, kwargs, _not_present) + if dim is _not_present or (single_dim and not isinstance(dim, Dim)): + with _enable_layers(self.dims): + print(f"dim fallback batch_tensor for {orig}") + return Tensor.from_batched( + orig(self._batchtensor, *args, **kwargs), self._has_device + ) + keepdim = ( + _getarg("keepdim", keepdim_offset, args, kwargs, False) if reduce else False + ) + t, levels = self._tensor, llist(self._levels) + dims = _dims(dim, self._batchtensor.ndim, keepdim, single_dim) + dim_indices = tuple(levels.index(d) for d in dims) + if reduce and not keepdim: + new_levels = [l for i, l in enumerate(levels) if i not in dim_indices] + else: + new_levels = levels + + if len(dim_indices) == 1: + dim_indices = dim_indices[ + 0 + ] # so that dims that really only take a single argument work... + args = list(args) + _patcharg(dim_name, dim_offset, args, kwargs, dim_indices) + + def wrap(t): + if isinstance(t, TensorLike): + return Tensor.from_positional(t, new_levels, self._has_device) + return t + + with _enable_layers(new_levels): + print(f"dim used batch_tensor for {orig}") + r = orig(t, *args, **kwargs) + return tree_map(wrap, r) + + return fn + + +def _def(name, *args, **kwargs): + from . import _Tensor + + orig = getattr(torch.Tensor, name) + setattr(_Tensor, name, _wrap(orig, *args, **kwargs)) + + +no_slice = slice(None) + +_orig_getitem = torch.Tensor.__getitem__ + + +class dim_tracker: + def __init__(self) -> None: + self.dims = llist() + self.count = [] + + def record(self, d): + if d not in self.dims: + self.dims.append(d) + self.count.append(1) + + def __getitem__(self, d): + return self.count[self.dims.index(d)] + + +def t__getitem__(self, input): + from . import _Tensor, Dim, DimensionBindError, DimList, Tensor, TensorLike + + # * bail to original example if we have a single non-Dim tensor, or a non-tensor + # * locate ... or an unbound tensor list, and determine its size, bind dim list + # (remember that None does not count to the total dim count) + # * bind simple dims and dim-packs to their sizes, count the number of uses of each dim, + # produce the re-view if needed + # * for each single-use dim index, replace with no_slice and mark that it will be added + # (keep track of whether we have to call super) + # * call super if needed + # * if we have dims to bind, bind them (it will help if we eliminated ... and None before) + # this handles bool indexing handling, as well as some other simple cases. + + is_simple = ( + not isinstance(input, Dim) + and not isinstance(input, (tuple, list)) + and + # WAR for functorch bug where zero time tensors in getitem are not handled correctly. + not (isinstance(input, TensorLike) and input.ndim == 0) + ) + + if is_simple: + if isinstance(self, _Tensor): + return _Tensor.__torch_function__(_orig_getitem, None, (self, input)) + else: + return _orig_getitem(self, input) + + # can further optimize this case + if not isinstance(input, tuple): + input = [input] + else: + input = list(input) + + dims_indexed = 0 + expanding_object = None + dimlists = [] + for i, s in enumerate(input): + if s is ... or isinstance(s, DimList) and not s.is_bound: + if expanding_object is not None: + msg = ( + "at most one ... or unbound dimension list can exist in indexing list but" + f" found 2 at offsets {i} and {expanding_object}" + ) + raise DimensionBindError(msg) + expanding_object = i + + if isinstance(s, DimList): + dims_indexed += len(s) if s.is_bound else 0 + dimlists.append(i) + elif s is not None and s is not ...: + dims_indexed += 1 + + ndim = self.ndim + if dims_indexed > ndim: + raise IndexError( + f"at least {dims_indexed} indices were supplied but the tensor only has {ndim} dimensions." + ) + if expanding_object is not None: + expanding_ndims = ndim - dims_indexed + obj = input[expanding_object] + if obj is ...: + input[expanding_object : expanding_object + 1] = [ + no_slice + ] * expanding_ndims + else: + obj.bind_len(expanding_ndims) + # flatten the dimslists into the indexing + for i in reversed(dimlists): + input[i : i + 1] = input[i] + dims_indexed = 0 + requires_view = False + size = self.size() + view_sizes = [] + dims_seen = dim_tracker() + + def add_dims(t): + if not isinstance(t, _Tensor): + return + for d in t.dims: + dims_seen.record(d) + + add_dims(self) + dim_packs = [] + for i, idx in enumerate(input): + if idx is None: + input[i] = no_slice + view_sizes.append(1) + requires_view = True + else: + sz = size[dims_indexed] + if isinstance(idx, Dim): + idx.size = sz + dims_seen.record(idx) + view_sizes.append(sz) + elif isinstance(idx, (tuple, list)) and idx and isinstance(idx[0], Dim): + for d in idx: + dims_seen.record(idx) + _bind_dims_to_size(sz, idx, f"offset {i}") + view_sizes.extend(d.size for d in idx) + requires_view = True + dim_packs.append(i) + else: + add_dims(idx) + view_sizes.append(sz) + dims_indexed += 1 + if requires_view: + self = self.view(*view_sizes) + for i in reversed(dim_packs): + input[i : i + 1] = input[i] + + # currenty: + # input is flat, containing either Dim, or Tensor, or something valid for standard indexing + # self may have first-class dims as well. + + # to index: + # drop the first class dims from self, they just become direct indices of their positions + + # figure out the dimensions of the indexing tensors: union of all the dims in the tensors in the index. + # these dimensions will appear and need to be bound at the first place tensor occures + + if isinstance(self, _Tensor): + ptensor_self, levels = self._tensor, list(self._levels) + # indices to ptensor rather than self which has first-class dimensions + input_it = iter(input) + flat_inputs = [next(input_it) if isinstance(l, int) else l for l in levels] + has_device = self._has_device + to_pad = 0 + else: + ptensor_self, flat_inputs = self, input + to_pad = ptensor_self.ndim - len(flat_inputs) + has_device = True + + result_levels = [] + index_levels = [] + tensor_insert_point = None + to_expand = {} + requires_getindex = False + for i, inp in enumerate(flat_inputs): + if isinstance(inp, Dim) and dims_seen[inp] == 1: + flat_inputs[i] = no_slice + result_levels.append(inp) + elif isinstance(inp, TensorLike): + requires_getindex = True + if tensor_insert_point is None: + tensor_insert_point = len(result_levels) + ptensor, levels, _ = _tensor_levels(inp) + to_expand[i] = levels + flat_inputs[i] = ptensor + for l in levels: + if l not in index_levels: + index_levels.append(l) + else: + requires_getindex = True + result_levels.append(0) + + if tensor_insert_point is not None: + result_levels[tensor_insert_point:tensor_insert_point] = index_levels + + for i, levels in to_expand.items(): + flat_inputs[i] = _match_levels(flat_inputs[i], levels, index_levels) + + if requires_getindex: + result = _orig_getitem(ptensor_self, flat_inputs) + else: + result = ptensor_self + + next_positional = -1 + if to_pad > 0: + result_levels.extend([0] * to_pad) + for i, r in enumerate(reversed(result_levels)): + if isinstance(r, int): + result_levels[-1 - i] = next_positional + next_positional -= 1 + + return Tensor.from_positional(result, result_levels, has_device) + + +# XXX - dim is optional and can be the outer-most dimension... +def stack(tensors, new_dim, dim=0, out=None): + if isinstance(dim, int): + return torch.stack(tensors, dim, out).index(dim, new_dim) + index = None + if out is not None: + out, index = _positional_no_permute(out, dim, expand_dim=True) + ptensors = [] + for t in tensors: + pt, pi = _positional_no_permute(t, dim, expand_dim=True) + if index is not None and pi != index: + pt = pt.move_dim(pi, index) + else: + index = pi + ptensors.append(pt) + pr = torch.stack(ptensors, index, out=out) + return pr.index((index, index + 1), (new_dim, dim)) + + +_orig_split = torch.Tensor.split + + +def split(self, split_size_or_sections, dim=0): + from . import _Tensor, Dim + + if isinstance(split_size_or_sections, int) or any( + isinstance(t, int) for t in split_size_or_sections + ): + if isinstance(dim, Dim): + raise ValueError( + "when dim is specified as a Dim object, split sizes must also be dimensions." + ) + return _orig_split(self, split_size_or_sections, dim=dim) + + if isinstance(dim, Dim): + assert isinstance(self, _Tensor), f"Tensor does not have dimension {dim}" + self, dim = _positional_no_permute(self, dim) + + size = self.size(dim) + total_bound_size = 0 + unbound = [] + sizes = [] + for i, d in enumerate(split_size_or_sections): + if d.is_bound: + sizes.append(d.size) + total_bound_size += d.size + else: + sizes.append(0) + unbound.append(i) + + if unbound: + assert ( + total_bound_size <= size + ), f"result dimensions are larger than original: {total_bound_size} vs {size} ({split_size_or_sections})" + remaining_size = size - total_bound_size + chunk_size = -(-remaining_size // len(unbound)) + for u in unbound: + sz = min(chunk_size, remaining_size) + split_size_or_sections[u].size = sz + sizes[u] = sz + remaining_size -= sz + else: + assert ( + total_bound_size == size + ), f"result dimensions do not match original: {total_bound_size} vs {size} ({split_size_or_sections})" + return tuple( + t.index(dim, d) + for d, t in zip(split_size_or_sections, _orig_split(self, sizes, dim=dim)) + ) diff --git a/lib/python3.12/site-packages/functorch/dim/tree_map.py b/lib/python3.12/site-packages/functorch/dim/tree_map.py new file mode 100644 index 0000000000000000000000000000000000000000..3d2eae0582c85619e0d28e648af58cbd847eee97 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/tree_map.py @@ -0,0 +1,15 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from functorch._C import dim + + +tree_flatten = dim.tree_flatten + + +def tree_map(fn, tree): + vs, unflatten = tree_flatten(tree) + return unflatten(fn(v) for v in vs) diff --git a/lib/python3.12/site-packages/functorch/dim/wrap_type.py b/lib/python3.12/site-packages/functorch/dim/wrap_type.py new file mode 100644 index 0000000000000000000000000000000000000000..aae543b91a896e2d5cef3b03d725c3d91ce1305d --- /dev/null +++ b/lib/python3.12/site-packages/functorch/dim/wrap_type.py @@ -0,0 +1,72 @@ +# Copyright (c) Facebook, Inc. and its affiliates. +# All rights reserved. +# +# This source code is licensed under the BSD-style license found in the +# LICENSE file in the root directory of this source tree. + +from types import ( + BuiltinMethodType, + FunctionType, + GetSetDescriptorType, + MethodDescriptorType, + WrapperDescriptorType, +) + +from functorch._C import dim as _C + + +_wrap_method = _C._wrap_method + +FUNC_TYPES = ( + FunctionType, + MethodDescriptorType, + BuiltinMethodType, + WrapperDescriptorType, +) +PROPERTY_TYPES = (GetSetDescriptorType, property) + + +def _py_wrap_method(orig, __torch_function__): + def impl(*args, **kwargs): + return __torch_function__(orig, None, args, kwargs) + + return impl + + +def wrap_type(use_c, to_patch, pattern, __torch_function__): + if use_c: + wrap_method = _wrap_method + else: + wrap_method = _py_wrap_method + + all = {} + for t in reversed(pattern.mro()[:-1]): # skip object + all.update(t.__dict__) + + def wrap_attr(orig): + return property(wrap_method(orig.__get__, __torch_function__)) + + for name, obj in all.items(): + if name in ( + "__dict__", + "__new__", + "__init__", + "__repr__", + "__weakref__", + "__doc__", + "__module__", + "__dir__", + ): + continue + + # skip things that have been overloaded + # things that come from object like `__eq__` still need to be patched, however. + if hasattr(to_patch, name) and getattr(to_patch, name) is not getattr( + object, name, None + ): + continue + + if isinstance(obj, FUNC_TYPES): + setattr(to_patch, name, wrap_method(obj, __torch_function__)) + elif isinstance(obj, PROPERTY_TYPES): + setattr(to_patch, name, wrap_attr(obj)) diff --git a/lib/python3.12/site-packages/functorch/einops/__init__.py b/lib/python3.12/site-packages/functorch/einops/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7ac34f7a3722010fc0fde97fd1cd72e76fa88b7 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/einops/__init__.py @@ -0,0 +1,4 @@ +from .rearrange import rearrange + + +__all__ = ["rearrange"] diff --git a/lib/python3.12/site-packages/functorch/einops/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/einops/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b8121700d6e1c075656c8d42822d79b60e2d3dd6 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/einops/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/einops/__pycache__/_parsing.cpython-312.pyc b/lib/python3.12/site-packages/functorch/einops/__pycache__/_parsing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..492a3e41fefdb75f9830eee5e71a0e06b537951c Binary files /dev/null and b/lib/python3.12/site-packages/functorch/einops/__pycache__/_parsing.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/einops/__pycache__/rearrange.cpython-312.pyc b/lib/python3.12/site-packages/functorch/einops/__pycache__/rearrange.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..217fe740ca64ad7df2c080713e216ca221ae6957 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/einops/__pycache__/rearrange.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/einops/_parsing.py b/lib/python3.12/site-packages/functorch/einops/_parsing.py new file mode 100644 index 0000000000000000000000000000000000000000..ee69aa60d1a58e44dcdf888d8c5e1a568f0151d6 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/einops/_parsing.py @@ -0,0 +1,304 @@ +"""Adapted from https://github.com/arogozhnikov/einops/blob/36c7bb16e57d6e57f8f3050f9e07abdf3f00469f/einops/parsing.py. + +MIT License + +Copyright (c) 2018 Alex Rogozhnikov + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +""" + +from __future__ import annotations + +import keyword +import warnings +from typing import Collection, List, Mapping, Optional, Set, Tuple, Union + + +_ellipsis: str = "\u2026" # NB, this is a single unicode symbol. String is used as it is not a list, but can be iterated + + +class AnonymousAxis: + """Used by `ParsedExpression` to represent an axis with a size (> 1), but no associated identifier. + + Note: Different instances of this class are not equal to each other, even if they have the same value. + """ + + def __init__(self, value: str) -> None: + self.value = int(value) + if self.value < 1: + raise ValueError( + f"Anonymous axis should have positive length, not {self.value}" + ) + + def __repr__(self) -> str: + return f"{self.value}-axis" + + +class ParsedExpression: + """Structure containing information about one side of an `einops`-style pattern (e.g. 'b c (h w)').""" + + def __init__( + self, + expression: str, + *, + allow_underscore: bool = False, + allow_duplicates: bool = False, + ) -> None: + """Parse the expression and store relevant metadata. + + Args: + expression (str): the `einops`-pattern to parse + allow_underscore (bool): whether to allow axis identifier names to begin with an underscore + allow_duplicates (bool): whether to allow an identifier to appear more than once in the expression + """ + self.has_ellipsis: bool = False + self.has_ellipsis_parenthesized: Optional[bool] = None + self.identifiers: Set[Union[str, AnonymousAxis]] = set() + # that's axes like 2, 3, 4 or 5. Axes with size 1 are exceptional and replaced with empty composition + self.has_non_unitary_anonymous_axes: bool = False + # composition keeps structure of composite axes, see how different corner cases are handled in tests + self.composition: List[Union[List[Union[str, AnonymousAxis]], str]] = [] + if "." in expression: + if "..." not in expression: + raise ValueError( + "Expression may contain dots only inside ellipsis (...)" + ) + if str.count(expression, "...") != 1 or str.count(expression, ".") != 3: + raise ValueError( + "Expression may contain dots only inside ellipsis (...); only one ellipsis for tensor " + ) + expression = expression.replace("...", _ellipsis) + self.has_ellipsis = True + + bracket_group: Optional[List[Union[str, AnonymousAxis]]] = None + + def add_axis_name(x: str) -> None: + if x in self.identifiers: + if not (allow_underscore and x == "_") and not allow_duplicates: + raise ValueError( + f"Indexing expression contains duplicate dimension '{x}'" + ) + if x == _ellipsis: + self.identifiers.add(_ellipsis) + if bracket_group is None: + self.composition.append(_ellipsis) + self.has_ellipsis_parenthesized = False + else: + bracket_group.append(_ellipsis) + self.has_ellipsis_parenthesized = True + else: + is_number = str.isdecimal(x) + if is_number and int(x) == 1: + # handling the case of anonymous axis of length 1 + if bracket_group is None: + self.composition.append([]) + else: + pass # no need to think about 1s inside parenthesis + return + is_axis_name, reason = self.check_axis_name_return_reason( + x, allow_underscore=allow_underscore + ) + if not (is_number or is_axis_name): + raise ValueError(f"Invalid axis identifier: {x}\n{reason}") + axis_name: Union[str, AnonymousAxis] = ( + AnonymousAxis(x) if is_number else x + ) + self.identifiers.add(axis_name) + if is_number: + self.has_non_unitary_anonymous_axes = True + if bracket_group is None: + self.composition.append([axis_name]) + else: + bracket_group.append(axis_name) + + current_identifier = None + for char in expression: + if char in "() ": + if current_identifier is not None: + add_axis_name(current_identifier) + current_identifier = None + if char == "(": + if bracket_group is not None: + raise ValueError( + "Axis composition is one-level (brackets inside brackets not allowed)" + ) + bracket_group = [] + elif char == ")": + if bracket_group is None: + raise ValueError("Brackets are not balanced") + self.composition.append(bracket_group) + bracket_group = None + elif str.isalnum(char) or char in ["_", _ellipsis]: + if current_identifier is None: + current_identifier = char + else: + current_identifier += char + else: + raise ValueError(f"Unknown character '{char}'") + + if bracket_group is not None: + raise ValueError(f"Imbalanced parentheses in expression: '{expression}'") + if current_identifier is not None: + add_axis_name(current_identifier) + + @staticmethod + def check_axis_name_return_reason( + name: str, allow_underscore: bool = False + ) -> Tuple[bool, str]: + """Check if the given axis name is valid, and a message explaining why if not. + + Valid axes names are python identifiers except keywords, and should not start or end with an underscore. + + Args: + name (str): the axis name to check + allow_underscore (bool): whether axis names are allowed to start with an underscore + + Returns: + Tuple[bool, str]: whether the axis name is valid, a message explaining why if not + """ + if not str.isidentifier(name): + return False, "not a valid python identifier" + elif name[0] == "_" or name[-1] == "_": + if name == "_" and allow_underscore: + return True, "" + return False, "axis name should should not start or end with underscore" + else: + if keyword.iskeyword(name): + warnings.warn( + f"It is discouraged to use axes names that are keywords: {name}", + RuntimeWarning, + ) + if name in ["axis"]: + warnings.warn( + "It is discouraged to use 'axis' as an axis name and will raise an error in future", + FutureWarning, + ) + return True, "" + + @staticmethod + def check_axis_name(name: str) -> bool: + """Check if the name is a valid axis name. + + Args: + name (str): the axis name to check + + Returns: + bool: whether the axis name is valid + """ + is_valid, _ = ParsedExpression.check_axis_name_return_reason(name) + return is_valid + + +def parse_pattern( + pattern: str, axes_lengths: Mapping[str, int] +) -> Tuple[ParsedExpression, ParsedExpression]: + """Parse an `einops`-style pattern into a left-hand side and right-hand side `ParsedExpression` object. + + Args: + pattern (str): the `einops`-style rearrangement pattern + axes_lengths (Mapping[str, int]): any additional length specifications for dimensions + + Returns: + Tuple[ParsedExpression, ParsedExpression]: a tuple containing the left-hand side and right-hand side expressions + """ + # adapted from einops.einops._prepare_transformation_recipe + # https://github.com/arogozhnikov/einops/blob/230ac1526c1f42c9e1f7373912c7f8047496df11/einops/einops.py + try: + left_str, right_str = pattern.split("->") + except ValueError: + raise ValueError("Pattern must contain a single '->' separator") from None + + if _ellipsis in axes_lengths: + raise ValueError(f"'{_ellipsis}' is not an allowed axis identifier") + + left = ParsedExpression(left_str) + right = ParsedExpression(right_str) + + if not left.has_ellipsis and right.has_ellipsis: + raise ValueError( + f"Ellipsis found in right side, but not left side of a pattern {pattern}" + ) + if left.has_ellipsis and left.has_ellipsis_parenthesized: + raise ValueError( + f"Ellipsis is parenthesis in the left side is not allowed: {pattern}" + ) + + return left, right + + +def validate_rearrange_expressions( + left: ParsedExpression, right: ParsedExpression, axes_lengths: Mapping[str, int] +) -> None: + """Perform expression validations that are specific to the `rearrange` operation. + + Args: + left (ParsedExpression): left-hand side expression + right (ParsedExpression): right-hand side expression + axes_lengths (Mapping[str, int]): any additional length specifications for dimensions + """ + for length in axes_lengths.values(): + if (length_type := type(length)) is not int: + raise TypeError( + f"rearrange axis lengths must be integers, got: {length_type}" + ) + + if left.has_non_unitary_anonymous_axes or right.has_non_unitary_anonymous_axes: + raise ValueError("rearrange only supports unnamed axes of size 1") + + difference = set.symmetric_difference(left.identifiers, right.identifiers) + if len(difference) > 0: + raise ValueError( + f"Identifiers only on one side of rearrange expression (should be on both): {difference}" + ) + + unmatched_axes = axes_lengths.keys() - left.identifiers + if len(unmatched_axes) > 0: + raise ValueError( + f"Identifiers not found in rearrange expression: {unmatched_axes}" + ) + + +def comma_separate(collection: Collection[Union[str, Collection[str]]]) -> str: + """Convert a collection of strings representing first class dims into a comma-separated string. + + Args: + collection (Collection[Union[str, Collection[str]]]): the collection of strings to convert + + Returns: + str: the comma-separated string + + Examples: + >>> comma_separate(("d0",)) + 'd0' + + >>> comma_separate(("d0", "d1", "d2", "d3")) + 'd0, d1, d2, d3' + + >>> comma_separate([("d1", "d4")]) + '(d1, d4)' + + >>> comma_separate([("d0",), (), ("d1",), ("d2",), ("d3", "d4")]) + '(d0,), (), (d1,), (d2,), (d3, d4)' + """ + return ", ".join( + item + if isinstance(item, str) + else f"({comma_separate(item)}{',' if len(item) == 1 else ''})" + for item in collection + ) diff --git a/lib/python3.12/site-packages/functorch/einops/rearrange.py b/lib/python3.12/site-packages/functorch/einops/rearrange.py new file mode 100644 index 0000000000000000000000000000000000000000..a0bceed7388347faefe35b14e1d31759fdb5fa5f --- /dev/null +++ b/lib/python3.12/site-packages/functorch/einops/rearrange.py @@ -0,0 +1,208 @@ +from __future__ import annotations + +import functools +from typing import Callable, Dict, List, Sequence, Tuple, Union + +import torch +from functorch._C import dim as _C + +from ._parsing import ( + _ellipsis, + AnonymousAxis, + comma_separate, + parse_pattern, + validate_rearrange_expressions, +) + + +__all__ = ["rearrange"] + +dims = _C.dims + + +@functools.lru_cache(256) +def _create_rearrange_callable( + tensor_ndim: int, pattern: str, **axes_lengths: int +) -> Callable[[torch.Tensor], torch.Tensor]: + r"""Translate an `einops`-style pattern into a callable that performs the rearrange using first-class dimensions. + + Since the an equivalent result is computed for tensors with the same number of dimensions, with the same pattern and + specified axes lengths, this function can be memoized. + + Args: + tensor_ndim (int): the number of dimensions in the tensor to rearrange + pattern (str): the `einops`-style rearrangement pattern + axes_lengths (int): any additional length specifications for dimensions + + Returns: + Callable[[torch.Tensor], torch.Tensor]: a callable that performs the rearrangement + """ + left, right = parse_pattern(pattern, axes_lengths) + validate_rearrange_expressions(left, right, axes_lengths) + + n_anon_dims = sum(not dim for dim in left.composition) + if left.has_ellipsis: + n_ellipsis_dims = tensor_ndim - (len(left.composition) - 1) + n_named_dims = len(left.identifiers) - 1 + + if (pattern_ndim := n_anon_dims + n_named_dims) > tensor_ndim: + raise ValueError( + f"Number of dimensions in pattern ({pattern_ndim}) must be less than or equal to the number of " + f"dimensions in the tensor ({tensor_ndim})" + ) + else: + n_ellipsis_dims = 0 + n_named_dims = len(left.identifiers) + + if (pattern_ndim := len(left.composition)) != tensor_ndim: + raise ValueError( + f"Number of dimensions in pattern ({pattern_ndim}) must be equal to the number of dimensions in " + f"the tensor ({tensor_ndim})" + ) + n_dims = n_named_dims + n_ellipsis_dims + n_anon_dims + + if n_dims == 0: + # an identity rearrangement on a 0-dimension tensor + return lambda tensor: tensor + + first_class_dims: Tuple[str, ...] = tuple(f"d{i}" for i in range(n_dims)) + identifier_dim_map: Dict[Union[str, AnonymousAxis], Tuple[str, ...]] = {} + anon_axes: List[AnonymousAxis] = [] + + # map the left-hand side identifiers to strings representing first class dims + dims_i = 0 + for dimension in left.composition: + if isinstance(dimension, list): + for identifier in dimension: + # non-unitary anon axes are not allowed in rearrange & unitary anon axes are represented as empty lists + assert isinstance(identifier, str) + identifier_dim_map[identifier] = (first_class_dims[dims_i],) + dims_i += 1 + if not dimension: + # unitary anonymous axis + anon_axis = AnonymousAxis("1") + identifier_dim_map[anon_axis] = (first_class_dims[dims_i],) + anon_axes.append(anon_axis) + dimension.append(anon_axis) + dims_i += 1 + elif dimension == _ellipsis: + identifier = _ellipsis + identifier_dim_map[identifier] = tuple( + first_class_dims[dims_i + j] for j in range(n_ellipsis_dims) + ) + dims_i += n_ellipsis_dims + else: + raise ValueError(f"Unexpected dimension: {dimension}") + + def composition_to_dims( + composition: Sequence[Union[List[Union[str, AnonymousAxis]], str]], + ) -> List[Union[str, Tuple[str, ...]]]: + """Convert a `ParsedExpression.composition` into a `Tensor.__getitem__` index of strings representing first + class dims.""" + dim_composition: List[Union[str, Tuple[str, ...]]] = [] + for dimension in composition: + if isinstance(dimension, list): + dim_composition.append( + tuple( + dim + for identifier in dimension + for dim in identifier_dim_map[identifier] + ) + ) + elif dimension == _ellipsis: + dim_composition.extend(identifier_dim_map[_ellipsis]) + else: + raise ValueError(f"Unexpected dimension: {dimension}") + return dim_composition + + left_dims = composition_to_dims(left.composition) + right_dims = composition_to_dims(right.composition) + anon_dims = tuple(identifier_dim_map[axis][0] for axis in anon_axes) + specified_lengths = tuple( + (identifier_dim_map[axis][0], length) for axis, length in axes_lengths.items() + ) + + custom_rearrange_callable_name = "do_rearrange" + custom_rearrange_callable_code = ( + ( + f"def {custom_rearrange_callable_name}(tensor):\n" + f" {comma_separate(first_class_dims)} = dims({n_dims})\n" + ) + + ( + "".join( + f" {dim}.size = {length}\n" for (dim, length) in specified_lengths + ) + if specified_lengths + else "" + ) + + f" tensor = tensor[{comma_separate(left_dims)}].order({comma_separate(right_dims)})\n" + + ( + f" return tensor.sum({comma_separate([anon_dims])}, keepdim=False)\n" + if anon_dims + else " return tensor\n" + ) + ) + + exec(custom_rearrange_callable_code) + return locals()[custom_rearrange_callable_name] + + +def rearrange( + tensor: Union[torch.Tensor, List[torch.Tensor], Tuple[torch.Tensor, ...]], + pattern: str, + **axes_lengths: int, +) -> torch.Tensor: + r"""A native implementation of `einops.rearrange`, a reader-friendly smart element reordering for multidimensional + tensors. This operation includes functionality of transpose (axes permutation), reshape (view), squeeze, unsqueeze, + stack, concatenate and other operations. + + See: https://einops.rocks/api/rearrange/ + + Args: + tensor (Tensor or sequence of Tensor): the tensor(s) to rearrange + pattern (str): the rearrangement pattern + axes_lengths (int): any additional length specifications for dimensions + + Returns: + Tensor: the rearranged tensor + + Examples: + >>> # suppose we have a set of 32 images in "h w c" format (height-width-channel) + >>> images = torch.randn((32, 30, 40, 3)) + + >>> # stack along first (batch) axis, output is a single array + >>> rearrange(images, "b h w c -> b h w c").shape + torch.Size([32, 30, 40, 3]) + + >>> # concatenate images along height (vertical axis), 960 = 32 * 30 + >>> rearrange(images, "b h w c -> (b h) w c").shape + torch.Size([960, 40, 3]) + + >>> # concatenated images along horizontal axis, 1280 = 32 * 40 + >>> rearrange(images, "b h w c -> h (b w) c").shape + torch.Size([30, 1280, 3]) + + >>> # reordered axes to "b c h w" format for deep learning + >>> rearrange(images, "b h w c -> b c h w").shape + torch.Size([32, 3, 30, 40]) + + >>> # flattened each image into a vector, 3600 = 30 * 40 * 3 + >>> rearrange(images, "b h w c -> b (c h w)").shape + torch.Size([32, 3600]) + + >>> # split each image into 4 smaller (top-left, top-right, bottom-left, bottom-right), 128 = 32 * 2 * 2 + >>> rearrange(images, "b (h1 h) (w1 w) c -> (b h1 w1) h w c", h1=2, w1=2).shape + torch.Size([128, 15, 20, 3]) + + >>> # space-to-depth operation + >>> rearrange(images, "b (h h1) (w w1) c -> b h w (c h1 w1)", h1=2, w1=2).shape + torch.Size([32, 15, 20, 12]) + """ + if not isinstance(tensor, torch.Tensor): + tensor = torch.stack(tensor) + + rearrange_callable = _create_rearrange_callable( + tensor.ndim, pattern, **axes_lengths + ) + + return rearrange_callable(tensor) diff --git a/lib/python3.12/site-packages/functorch/experimental/__init__.py b/lib/python3.12/site-packages/functorch/experimental/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3941f6d96e1f6df532966d06de4f72a952a58465 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/experimental/__init__.py @@ -0,0 +1,5 @@ +# PyTorch forward-mode is not mature yet +from functorch import functionalize +from torch._functorch.apis import chunk_vmap +from torch._functorch.batch_norm_replacement import replace_all_batch_norm_modules_ +from torch._functorch.eager_transforms import hessian, jacfwd, jvp diff --git a/lib/python3.12/site-packages/functorch/experimental/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/functorch/experimental/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73ee77610787b00fab1c3f436dcd988104d7f7ea Binary files /dev/null and b/lib/python3.12/site-packages/functorch/experimental/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/experimental/__pycache__/control_flow.cpython-312.pyc b/lib/python3.12/site-packages/functorch/experimental/__pycache__/control_flow.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eae20391e639259d2b8c7e6b0ad07e676a7b5e00 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/experimental/__pycache__/control_flow.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/experimental/__pycache__/ops.cpython-312.pyc b/lib/python3.12/site-packages/functorch/experimental/__pycache__/ops.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7c4a2bb4062566b5c24458aea75ccd99b98a1d9 Binary files /dev/null and b/lib/python3.12/site-packages/functorch/experimental/__pycache__/ops.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/functorch/experimental/control_flow.py b/lib/python3.12/site-packages/functorch/experimental/control_flow.py new file mode 100644 index 0000000000000000000000000000000000000000..cbfd76d184cc224042a1ee19454d6957c2221b3b --- /dev/null +++ b/lib/python3.12/site-packages/functorch/experimental/control_flow.py @@ -0,0 +1,7 @@ +from torch import cond # noqa: F401 +from torch._higher_order_ops.cond import UnsupportedAliasMutationException # noqa: F401 +from torch._higher_order_ops.map import ( # noqa: F401 + _stack_pytree, + _unstack_pytree, + map, +) diff --git a/lib/python3.12/site-packages/functorch/experimental/ops.py b/lib/python3.12/site-packages/functorch/experimental/ops.py new file mode 100644 index 0000000000000000000000000000000000000000..7a502ef2b002cd824e7b67d08fccac872b313110 --- /dev/null +++ b/lib/python3.12/site-packages/functorch/experimental/ops.py @@ -0,0 +1 @@ +from torch._ops import HigherOrderOperator # noqa: F401 diff --git a/lib/python3.12/site-packages/jedi-0.19.2.dist-info/AUTHORS.txt b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/AUTHORS.txt new file mode 100644 index 0000000000000000000000000000000000000000..3ccb0a7cf31cf5d6bdcfa923607bddbb4e882e32 --- /dev/null +++ b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/AUTHORS.txt @@ -0,0 +1,72 @@ +Main Authors +------------ + +- David Halter (@davidhalter) +- Takafumi Arakaki (@tkf) + +Code Contributors +----------------- + +- Danilo Bargen (@dbrgn) +- Laurens Van Houtven (@lvh) <_@lvh.cc> +- Aldo Stracquadanio (@Astrac) +- Jean-Louis Fuchs (@ganwell) +- tek (@tek) +- Yasha Borevich (@jjay) +- Aaron Griffin +- andviro (@andviro) +- Mike Gilbert (@floppym) +- Aaron Meurer (@asmeurer) +- Lubos Trilety +- Akinori Hattori (@hattya) +- srusskih (@srusskih) +- Steven Silvester (@blink1073) +- Colin Duquesnoy (@ColinDuquesnoy) +- Jorgen Schaefer (@jorgenschaefer) +- Fredrik Bergroth (@fbergroth) +- Mathias Fußenegger (@mfussenegger) +- Syohei Yoshida (@syohex) +- ppalucky (@ppalucky) +- immerrr (@immerrr) immerrr@gmail.com +- Albertas Agejevas (@alga) +- Savor d'Isavano (@KenetJervet) +- Phillip Berndt (@phillipberndt) +- Ian Lee (@IanLee1521) +- Farkhad Khatamov (@hatamov) +- Kevin Kelley (@kelleyk) +- Sid Shanker (@squidarth) +- Reinoud Elhorst (@reinhrst) +- Guido van Rossum (@gvanrossum) +- Dmytro Sadovnychyi (@sadovnychyi) +- Cristi Burcă (@scribu) +- bstaint (@bstaint) +- Mathias Rav (@Mortal) +- Daniel Fiterman (@dfit99) +- Simon Ruggier (@sruggier) +- Élie Gouzien (@ElieGouzien) +- Robin Roth (@robinro) +- Malte Plath (@langsamer) +- Anton Zub (@zabulazza) +- Maksim Novikov (@m-novikov) +- Tobias Rzepka (@TobiasRzepka) +- micbou (@micbou) +- Dima Gerasimov (@karlicoss) +- Max Woerner Chase (@mwchase) +- Johannes Maria Frank (@jmfrank63) +- Shane Steinert-Threlkeld (@shanest) +- Tim Gates (@timgates42) +- Lior Goldberg (@goldberglior) +- Ryan Clary (@mrclary) +- Max Mäusezahl (@mmaeusezahl) +- Vladislav Serebrennikov (@endilll) +- Andrii Kolomoiets (@muffinmad) +- Leo Ryu (@Leo-Ryu) +- Joseph Birkner (@josephbirkner) +- Márcio Mazza (@marciomazza) +- Martin Vielsmaier (@moser) +- TingJia Wu (@WutingjiaX) +- Nguyễn Hồng Quân + +And a few more "anonymous" contributors. + +Note: (@user) means a github user name. diff --git a/lib/python3.12/site-packages/jedi-0.19.2.dist-info/INSTALLER b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/jedi-0.19.2.dist-info/LICENSE.txt b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/LICENSE.txt new file mode 100644 index 0000000000000000000000000000000000000000..94f954567b1c4a33c601b6bf3688ec08b695cbc0 --- /dev/null +++ b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/LICENSE.txt @@ -0,0 +1,24 @@ +All contributions towards Jedi are MIT licensed. + +------------------------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) <2013> + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lib/python3.12/site-packages/jedi-0.19.2.dist-info/METADATA b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..296141ae5512160118f41acf740157e1bdcbb264 --- /dev/null +++ b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/METADATA @@ -0,0 +1,623 @@ +Metadata-Version: 2.1 +Name: jedi +Version: 0.19.2 +Summary: An autocompletion tool for Python that can be used for text editors. +Home-page: https://github.com/davidhalter/jedi +Author: David Halter +Author-email: davidhalter88@gmail.com +Maintainer: David Halter +Maintainer-email: davidhalter88@gmail.com +License: MIT +Project-URL: Documentation, https://jedi.readthedocs.io/en/latest/ +Keywords: python completion refactoring vim +Platform: any +Classifier: Development Status :: 4 - Beta +Classifier: Environment :: Plugins +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: MIT License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Editors :: Integrated Development Environments (IDE) +Classifier: Topic :: Utilities +Requires-Python: >=3.6 +Requires-Dist: parso (<0.9.0,>=0.8.4) +Provides-Extra: docs +Requires-Dist: Jinja2 (==2.11.3) ; extra == 'docs' +Requires-Dist: MarkupSafe (==1.1.1) ; extra == 'docs' +Requires-Dist: Pygments (==2.8.1) ; extra == 'docs' +Requires-Dist: alabaster (==0.7.12) ; extra == 'docs' +Requires-Dist: babel (==2.9.1) ; extra == 'docs' +Requires-Dist: chardet (==4.0.0) ; extra == 'docs' +Requires-Dist: commonmark (==0.8.1) ; extra == 'docs' +Requires-Dist: docutils (==0.17.1) ; extra == 'docs' +Requires-Dist: future (==0.18.2) ; extra == 'docs' +Requires-Dist: idna (==2.10) ; extra == 'docs' +Requires-Dist: imagesize (==1.2.0) ; extra == 'docs' +Requires-Dist: mock (==1.0.1) ; extra == 'docs' +Requires-Dist: packaging (==20.9) ; extra == 'docs' +Requires-Dist: pyparsing (==2.4.7) ; extra == 'docs' +Requires-Dist: pytz (==2021.1) ; extra == 'docs' +Requires-Dist: readthedocs-sphinx-ext (==2.1.4) ; extra == 'docs' +Requires-Dist: recommonmark (==0.5.0) ; extra == 'docs' +Requires-Dist: requests (==2.25.1) ; extra == 'docs' +Requires-Dist: six (==1.15.0) ; extra == 'docs' +Requires-Dist: snowballstemmer (==2.1.0) ; extra == 'docs' +Requires-Dist: sphinx-rtd-theme (==0.4.3) ; extra == 'docs' +Requires-Dist: sphinx (==1.8.5) ; extra == 'docs' +Requires-Dist: sphinxcontrib-serializinghtml (==1.1.4) ; extra == 'docs' +Requires-Dist: sphinxcontrib-websupport (==1.2.4) ; extra == 'docs' +Requires-Dist: urllib3 (==1.26.4) ; extra == 'docs' +Provides-Extra: qa +Requires-Dist: flake8 (==5.0.4) ; extra == 'qa' +Requires-Dist: mypy (==0.971) ; extra == 'qa' +Requires-Dist: types-setuptools (==67.2.0.1) ; extra == 'qa' +Provides-Extra: testing +Requires-Dist: Django ; extra == 'testing' +Requires-Dist: attrs ; extra == 'testing' +Requires-Dist: colorama ; extra == 'testing' +Requires-Dist: docopt ; extra == 'testing' +Requires-Dist: pytest (<9.0.0) ; extra == 'testing' + +#################################################################################### +Jedi - an awesome autocompletion, static analysis and refactoring library for Python +#################################################################################### + +.. image:: http://isitmaintained.com/badge/open/davidhalter/jedi.svg + :target: https://github.com/davidhalter/jedi/issues + :alt: The percentage of open issues and pull requests + +.. image:: http://isitmaintained.com/badge/resolution/davidhalter/jedi.svg + :target: https://github.com/davidhalter/jedi/issues + :alt: The resolution time is the median time an issue or pull request stays open. + +.. image:: https://github.com/davidhalter/jedi/workflows/ci/badge.svg?branch=master + :target: https://github.com/davidhalter/jedi/actions + :alt: Tests + +.. image:: https://pepy.tech/badge/jedi + :target: https://pepy.tech/project/jedi + :alt: PyPI Downloads + + +Jedi is a static analysis tool for Python that is typically used in +IDEs/editors plugins. Jedi has a focus on autocompletion and goto +functionality. Other features include refactoring, code search and finding +references. + +Jedi has a simple API to work with. There is a reference implementation as a +`VIM-Plugin `_. Autocompletion in your +REPL is also possible, IPython uses it natively and for the CPython REPL you +can install it. Jedi is well tested and bugs should be rare. + +Jedi can currently be used with the following editors/projects: + +- Vim (jedi-vim_, YouCompleteMe_, deoplete-jedi_, completor.vim_) +- `Visual Studio Code`_ (via `Python Extension `_) +- Emacs (Jedi.el_, company-mode_, elpy_, anaconda-mode_, ycmd_) +- Sublime Text (SublimeJEDI_ [ST2 + ST3], anaconda_ [only ST3]) +- TextMate_ (Not sure if it's actually working) +- Kate_ version 4.13+ supports it natively, you have to enable it, though. [`see + `_] +- Atom_ (autocomplete-python-jedi_) +- `GNOME Builder`_ (with support for GObject Introspection) +- Gedit (gedi_) +- wdb_ - Web Debugger +- `Eric IDE`_ +- `IPython 6.0.0+ `_ +- `xonsh shell `_ has `jedi extension `_ + +and many more! + +There are a few language servers that use Jedi: + +- `jedi-language-server `_ +- `python-language-server `_ (currently unmaintained) +- `python-lsp-server `_ (fork from python-language-server) +- `anakin-language-server `_ + +Here are some pictures taken from jedi-vim_: + +.. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_complete.png + +Completion for almost anything: + +.. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_function.png + +Documentation: + +.. image:: https://github.com/davidhalter/jedi/raw/master/docs/_screenshots/screenshot_pydoc.png + + +Get the latest version from `github `_ +(master branch should always be kind of stable/working). + +Docs are available at `https://jedi.readthedocs.org/en/latest/ +`_. Pull requests with enhancements +and/or fixes are awesome and most welcome. Jedi uses `semantic versioning +`_. + +If you want to stay **up-to-date** with releases, please **subscribe** to this +mailing list: https://groups.google.com/g/jedi-announce. To subscribe you can +simply send an empty email to ``jedi-announce+subscribe@googlegroups.com``. + +Issues & Questions +================== + +You can file issues and questions in the `issue tracker +`. Alternatively you can also ask on +`Stack Overflow `_ with +the label ``python-jedi``. + +Installation +============ + +`Check out the docs `_. + +Features and Limitations +======================== + +Jedi's features are listed here: +`Features `_. + +You can run Jedi on Python 3.6+ but it should also +understand code that is older than those versions. Additionally you should be +able to use `Virtualenvs `_ +very well. + +Tips on how to use Jedi efficiently can be found `here +`_. + +API +--- + +You can find a comprehensive documentation for the +`API here `_. + +Autocompletion / Goto / Documentation +------------------------------------- + +There are the following commands: + +- ``jedi.Script.goto`` +- ``jedi.Script.infer`` +- ``jedi.Script.help`` +- ``jedi.Script.complete`` +- ``jedi.Script.get_references`` +- ``jedi.Script.get_signatures`` +- ``jedi.Script.get_context`` + +The returned objects are very powerful and are really all you might need. + +Autocompletion in your REPL (IPython, etc.) +------------------------------------------- + +Jedi is a dependency of IPython. Autocompletion in IPython with Jedi is +therefore possible without additional configuration. + +Here is an `example video `_ how REPL completion +can look like. +For the ``python`` shell you can enable tab completion in a `REPL +`_. + +Static Analysis +--------------- + +For a lot of forms of static analysis, you can try to use +``jedi.Script(...).get_names``. It will return a list of names that you can +then filter and work with. There is also a way to list the syntax errors in a +file: ``jedi.Script.get_syntax_errors``. + + +Refactoring +----------- + +Jedi supports the following refactorings: + +- ``jedi.Script.inline`` +- ``jedi.Script.rename`` +- ``jedi.Script.extract_function`` +- ``jedi.Script.extract_variable`` + +Code Search +----------- + +There is support for module search with ``jedi.Script.search``, and project +search for ``jedi.Project.search``. The way to search is either by providing a +name like ``foo`` or by using dotted syntax like ``foo.bar``. Additionally you +can provide the API type like ``class foo.bar.Bar``. There are also the +functions ``jedi.Script.complete_search`` and ``jedi.Project.complete_search``. + +Development +=========== + +There's a pretty good and extensive `development documentation +`_. + +Testing +======= + +The test suite uses ``pytest``:: + + pip install pytest + +If you want to test only a specific Python version (e.g. Python 3.8), it is as +easy as:: + + python3.8 -m pytest + +For more detailed information visit the `testing documentation +`_. + +Acknowledgements +================ + +Thanks a lot to all the +`contributors `_! + + +.. _jedi-vim: https://github.com/davidhalter/jedi-vim +.. _youcompleteme: https://github.com/ycm-core/YouCompleteMe +.. _deoplete-jedi: https://github.com/zchee/deoplete-jedi +.. _completor.vim: https://github.com/maralla/completor.vim +.. _Jedi.el: https://github.com/tkf/emacs-jedi +.. _company-mode: https://github.com/syohex/emacs-company-jedi +.. _elpy: https://github.com/jorgenschaefer/elpy +.. _anaconda-mode: https://github.com/proofit404/anaconda-mode +.. _ycmd: https://github.com/abingham/emacs-ycmd +.. _sublimejedi: https://github.com/srusskih/SublimeJEDI +.. _anaconda: https://github.com/DamnWidget/anaconda +.. _wdb: https://github.com/Kozea/wdb +.. _TextMate: https://github.com/lawrenceakka/python-jedi.tmbundle +.. _Kate: https://kate-editor.org +.. _Atom: https://atom.io/ +.. _autocomplete-python-jedi: https://atom.io/packages/autocomplete-python-jedi +.. _GNOME Builder: https://wiki.gnome.org/Apps/Builder +.. _Visual Studio Code: https://code.visualstudio.com/ +.. _gedi: https://github.com/isamert/gedi +.. _Eric IDE: https://eric-ide.python-projects.org + + +.. :changelog: + +Changelog +--------- + +Unreleased +++++++++++ + +0.19.2 (2024-11-10) ++++++++++++++++++++ + +- Python 3.13 support + +0.19.1 (2023-10-02) ++++++++++++++++++++ + +- Python 3.12 support (Thanks Peter!) + +0.19.0 (2023-07-29) ++++++++++++++++++++ + +- Python 3.11 support +- Massive improvements in performance for ``Interpreter`` (e.g. IPython) users. + This especially affects ``pandas`` users with large datasets. +- Add ``jedi.settings.allow_unsafe_interpreter_executions`` to make it easier + for IPython users to avoid unsafe executions. + +0.18.2 (2022-11-21) ++++++++++++++++++++ + +- Added dataclass-equivalent for attrs.define +- Find fixtures from Pytest entrypoints; Examples of pytest plugins installed + like this are pytest-django, pytest-sugar and Faker. +- Fixed Project.search, when a venv was involved, which is why for example + `:Pyimport django.db` did not work in some cases in jedi-vim. +- And many smaller bugfixes + +0.18.1 (2021-11-17) ++++++++++++++++++++ + +- Implict namespaces are now a separate types in ``Name().type`` +- Python 3.10 support +- Mostly bugfixes + +0.18.0 (2020-12-25) ++++++++++++++++++++ + +- Dropped Python 2 and Python 3.5 +- Using ``pathlib.Path()`` as an output instead of ``str`` in most places: + - ``Project.path`` + - ``Script.path`` + - ``Definition.module_path`` + - ``Refactoring.get_renames`` + - ``Refactoring.get_changed_files`` +- Functions with ``@property`` now return ``property`` instead of ``function`` + in ``Name().type`` +- Started using annotations +- Better support for the walrus operator +- Project attributes are now read accessible +- Removed all deprecations + +This is likely going to be the last minor release before 1.0. + +0.17.2 (2020-07-17) ++++++++++++++++++++ + +- Added an option to pass environment variables to ``Environment`` +- ``Project(...).path`` exists now +- Support for Python 3.9 +- A few bugfixes + +This will be the last release that supports Python 2 and Python 3.5. +``0.18.0`` will be Python 3.6+. + +0.17.1 (2020-06-20) ++++++++++++++++++++ + +- Django ``Model`` meta class support +- Django Manager support (completion on Managers/QuerySets) +- Added Django Stubs to Jedi, thanks to all contributors of the + `Django Stubs `_ project +- Added ``SyntaxError.get_message`` +- Python 3.9 support +- Bugfixes (mostly towards Generics) + +0.17.0 (2020-04-14) ++++++++++++++++++++ + +- Added ``Project`` support. This allows a user to specify which folders Jedi + should work with. +- Added support for Refactoring. The following refactorings have been + implemented: ``Script.rename``, ``Script.inline``, + ``Script.extract_variable`` and ``Script.extract_function``. +- Added ``Script.get_syntax_errors`` to display syntax errors in the current + script. +- Added code search capabilities both for individual files and projects. The + new functions are ``Project.search``, ``Project.complete_search``, + ``Script.search`` and ``Script.complete_search``. +- Added ``Script.help`` to make it easier to display a help window to people. + Now returns pydoc information as well for Python keywords/operators. This + means that on the class keyword it will now return the docstring of Python's + builtin function ``help('class')``. +- The API documentation is now way more readable and complete. Check it out + under https://jedi.readthedocs.io. A lot of it has been rewritten. +- Removed Python 3.4 support +- Many bugfixes + +This is likely going to be the last minor version that supports Python 2 and +Python3.5. Bugfixes will be provided in 0.17.1+. The next minor/major version +will probably be Jedi 1.0.0. + +0.16.0 (2020-01-26) ++++++++++++++++++++ + +- **Added** ``Script.get_context`` to get information where you currently are. +- Completions/type inference of **Pytest fixtures**. +- Tensorflow, Numpy and Pandas completions should now be about **4-10x faster** + after the first time they are used. +- Dict key completions are working now. e.g. ``d = {1000: 3}; d[10`` will + expand to ``1000``. +- Completion for "proxies" works now. These are classes that have a + ``__getattr__(self, name)`` method that does a ``return getattr(x, name)``. + after loading them initially. +- Goto on a function/attribute in a class now goes to the definition in its + super class. +- Big **Script API Changes**: + - The line and column parameters of ``jedi.Script`` are now deprecated + - ``completions`` deprecated, use ``complete`` instead + - ``goto_assignments`` deprecated, use ``goto`` instead + - ``goto_definitions`` deprecated, use ``infer`` instead + - ``call_signatures`` deprecated, use ``get_signatures`` instead + - ``usages`` deprecated, use ``get_references`` instead + - ``jedi.names`` deprecated, use ``jedi.Script(...).get_names()`` +- ``BaseName.goto_assignments`` renamed to ``BaseName.goto`` +- Add follow_imports to ``Name.goto``. Now its signature matches + ``Script.goto``. +- **Python 2 support deprecated**. For this release it is best effort. Python 2 + has reached the end of its life and now it's just about a smooth transition. + Bugs for Python 2 will not be fixed anymore and a third of the tests are + already skipped. +- Removed ``settings.no_completion_duplicates``. It wasn't tested and nobody + was probably using it anyway. +- Removed ``settings.use_filesystem_cache`` and + ``settings.additional_dynamic_modules``, they have no usage anymore. Pretty + much nobody was probably using them. + +0.15.2 (2019-12-20) ++++++++++++++++++++ + +- Signatures are now detected a lot better +- Add fuzzy completions with ``Script(...).completions(fuzzy=True)`` +- Files bigger than one MB (about 20kLOC) get cropped to avoid getting + stuck completely. +- Many small Bugfixes +- A big refactoring around contexts/values + +0.15.1 (2019-08-13) ++++++++++++++++++++ + +- Small bugfix and removal of a print statement + +0.15.0 (2019-08-11) ++++++++++++++++++++ + +- Added file path completions, there's a **new** ``Completion.type`` now: + ``path``. Example: ``'/ho`` -> ``'/home/`` +- ``*args``/``**kwargs`` resolving. If possible Jedi replaces the parameters + with the actual alternatives. +- Better support for enums/dataclasses +- When using Interpreter, properties are now executed, since a lot of people + have complained about this. Discussion in #1299, #1347. + +New APIs: + +- ``Name.get_signatures() -> List[Signature]``. Signatures are similar to + ``CallSignature``. ``Name.params`` is therefore deprecated. +- ``Signature.to_string()`` to format signatures. +- ``Signature.params -> List[ParamName]``, ParamName has the + following additional attributes ``infer_default()``, ``infer_annotation()``, + ``to_string()``, and ``kind``. +- ``Name.execute() -> List[Name]``, makes it possible to infer + return values of functions. + + +0.14.1 (2019-07-13) ++++++++++++++++++++ + +- CallSignature.index should now be working a lot better +- A couple of smaller bugfixes + +0.14.0 (2019-06-20) ++++++++++++++++++++ + +- Added ``goto_*(prefer_stubs=True)`` as well as ``goto_*(prefer_stubs=True)`` +- Stubs are used now for type inference +- Typeshed is used for better type inference +- Reworked Name.full_name, should have more correct return values + +0.13.3 (2019-02-24) ++++++++++++++++++++ + +- Fixed an issue with embedded Python, see https://github.com/davidhalter/jedi-vim/issues/870 + +0.13.2 (2018-12-15) ++++++++++++++++++++ + +- Fixed a bug that led to Jedi spawning a lot of subprocesses. + +0.13.1 (2018-10-02) ++++++++++++++++++++ + +- Bugfixes, because tensorflow completions were still slow. + +0.13.0 (2018-10-02) ++++++++++++++++++++ + +- A small release. Some bug fixes. +- Remove Python 3.3 support. Python 3.3 support has been dropped by the Python + foundation. +- Default environments are now using the same Python version as the Python + process. In 0.12.x, we used to load the latest Python version on the system. +- Added ``include_builtins`` as a parameter to usages. +- ``goto_assignments`` has a new ``follow_builtin_imports`` parameter that + changes the previous behavior slightly. + +0.12.1 (2018-06-30) ++++++++++++++++++++ + +- This release forces you to upgrade parso. If you don't, nothing will work + anymore. Otherwise changes should be limited to bug fixes. Unfortunately Jedi + still uses a few internals of parso that make it hard to keep compatibility + over multiple releases. Parso >=0.3.0 is going to be needed. + +0.12.0 (2018-04-15) ++++++++++++++++++++ + +- Virtualenv/Environment support +- F-String Completion/Goto Support +- Cannot crash with segfaults anymore +- Cleaned up import logic +- Understand async/await and autocomplete it (including async generators) +- Better namespace completions +- Passing tests for Windows (including CI for Windows) +- Remove Python 2.6 support + +0.11.1 (2017-12-14) ++++++++++++++++++++ + +- Parso update - the caching layer was broken +- Better usages - a lot of internal code was ripped out and improved. + +0.11.0 (2017-09-20) ++++++++++++++++++++ + +- Split Jedi's parser into a separate project called ``parso``. +- Avoiding side effects in REPL completion. +- Numpy docstring support should be much better. +- Moved the `settings.*recursion*` away, they are no longer usable. + +0.10.2 (2017-04-05) ++++++++++++++++++++ + +- Python Packaging sucks. Some files were not included in 0.10.1. + +0.10.1 (2017-04-05) ++++++++++++++++++++ + +- Fixed a few very annoying bugs. +- Prepared the parser to be factored out of Jedi. + +0.10.0 (2017-02-03) ++++++++++++++++++++ + +- Actual semantic completions for the complete Python syntax. +- Basic type inference for ``yield from`` PEP 380. +- PEP 484 support (most of the important features of it). Thanks Claude! (@reinhrst) +- Added ``get_line_code`` to ``Name`` and ``Completion`` objects. +- Completely rewritten the type inference engine. +- A new and better parser for (fast) parsing diffs of Python code. + +0.9.0 (2015-04-10) +++++++++++++++++++ + +- The import logic has been rewritten to look more like Python's. There is now + an ``InferState.modules`` import cache, which resembles ``sys.modules``. +- Integrated the parser of 2to3. This will make refactoring possible. It will + also be possible to check for error messages (like compiling an AST would give) + in the future. +- With the new parser, the type inference also completely changed. It's now + simpler and more readable. +- Completely rewritten REPL completion. +- Added ``jedi.names``, a command to do static analysis. Thanks to that + sourcegraph guys for sponsoring this! +- Alpha version of the linter. + + +0.8.1 (2014-07-23) ++++++++++++++++++++ + +- Bugfix release, the last release forgot to include files that improve + autocompletion for builtin libraries. Fixed. + +0.8.0 (2014-05-05) ++++++++++++++++++++ + +- Memory Consumption for compiled modules (e.g. builtins, sys) has been reduced + drastically. Loading times are down as well (it takes basically as long as an + import). +- REPL completion is starting to become usable. +- Various small API changes. Generally this release focuses on stability and + refactoring of internal APIs. +- Introducing operator precedence, which makes calculating correct Array + indices and ``__getattr__`` strings possible. + +0.7.0 (2013-08-09) +++++++++++++++++++ + +- Switched from LGPL to MIT license. +- Added an Interpreter class to the API to make autocompletion in REPL + possible. +- Added autocompletion support for namespace packages. +- Add sith.py, a new random testing method. + +0.6.0 (2013-05-14) +++++++++++++++++++ + +- Much faster parser with builtin part caching. +- A test suite, thanks @tkf. + +0.5 versions (2012) ++++++++++++++++++++ + +- Initial development. + + diff --git a/lib/python3.12/site-packages/jedi-0.19.2.dist-info/RECORD b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..e8e7cab755d5502a5f4d1525170630e7ee8b8426 --- /dev/null +++ b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/RECORD @@ -0,0 +1,1965 @@ +jedi-0.19.2.dist-info/AUTHORS.txt,sha256=0LRys2YXtK-HSBZxM43WSROeXGGQkJN7oxfJi-0rv_I,2661 +jedi-0.19.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +jedi-0.19.2.dist-info/LICENSE.txt,sha256=eOYM0LjyhpTzAZVILDPXaQjYRrDRUnjetzMqoiuo5BI,1241 +jedi-0.19.2.dist-info/METADATA,sha256=NBVDMKX180YqwIBGDmQMyx47L7FFxBYWG9MrCbCYf_s,22937 +jedi-0.19.2.dist-info/RECORD,, +jedi-0.19.2.dist-info/WHEEL,sha256=kGT74LWyRUZrL4VgLh6_g12IeVl_9u9ZVhadrgXZUEY,110 +jedi-0.19.2.dist-info/top_level.txt,sha256=SqNj9U77yOn5Zf4Ji6CqL8-D2eNg_F71GHo8V8SMV04,5 +jedi/__init__.py,sha256=lzDn_PAg7XmOm2Al-JyKJVwd9Oqv0WGp3KXlQ7A6pCU,1486 +jedi/__main__.py,sha256=SONRAB5KoCcU3DRfUZhWWFG8n_fs7Rls-bzzM4VWqD0,1950 +jedi/__pycache__/__init__.cpython-312.pyc,, +jedi/__pycache__/__main__.cpython-312.pyc,, +jedi/__pycache__/_compatibility.cpython-312.pyc,, +jedi/__pycache__/cache.cpython-312.pyc,, +jedi/__pycache__/common.cpython-312.pyc,, +jedi/__pycache__/debug.cpython-312.pyc,, +jedi/__pycache__/file_io.cpython-312.pyc,, +jedi/__pycache__/parser_utils.cpython-312.pyc,, +jedi/__pycache__/settings.cpython-312.pyc,, +jedi/__pycache__/utils.cpython-312.pyc,, +jedi/_compatibility.py,sha256=ZScone1QgkbDJ03hdPaSS7dmAllDx8uJKvXRLBTzrCs,1491 +jedi/api/__init__.py,sha256=adUBV6tWtXe20XOaZ3i_5jBHJxmVmjcySe7k1z5Y5hY,32428 +jedi/api/__pycache__/__init__.cpython-312.pyc,, +jedi/api/__pycache__/classes.cpython-312.pyc,, +jedi/api/__pycache__/completion.cpython-312.pyc,, +jedi/api/__pycache__/completion_cache.cpython-312.pyc,, +jedi/api/__pycache__/environment.cpython-312.pyc,, +jedi/api/__pycache__/errors.cpython-312.pyc,, +jedi/api/__pycache__/exceptions.cpython-312.pyc,, +jedi/api/__pycache__/file_name.cpython-312.pyc,, +jedi/api/__pycache__/helpers.cpython-312.pyc,, +jedi/api/__pycache__/interpreter.cpython-312.pyc,, +jedi/api/__pycache__/keywords.cpython-312.pyc,, +jedi/api/__pycache__/project.cpython-312.pyc,, +jedi/api/__pycache__/replstartup.cpython-312.pyc,, +jedi/api/__pycache__/strings.cpython-312.pyc,, +jedi/api/classes.py,sha256=H1HBJY6bkCHOKKfvJ2AoKwG44CB820rWV78ZRFEf23M,29600 +jedi/api/completion.py,sha256=PM-W-ro5K0JRZMMnAbe91BAEW_Ky7vq_ZpVg45hFyMY,28379 +jedi/api/completion_cache.py,sha256=UOaBWoCsSLjyOiDEWHlUwgDiMCY2Rn2J2zMeI_s5lCM,954 +jedi/api/environment.py,sha256=fPOjUHjDBdAKkF39vZSgFtUwCl7NN1glN5zvm3jtJcg,17254 +jedi/api/errors.py,sha256=jvJvx0Sk8mWLQagxZlZuDszOrQ8DYenfViZrlgWUdeU,1253 +jedi/api/exceptions.py,sha256=RphZJj59VqGJnfw8XUVJ_VU-5N3nwgOk08GADNflxPY,990 +jedi/api/file_name.py,sha256=xlevCfp9B5hUsR94AH09tg4Qn1YXkGxM92uQPf8WpTw,5620 +jedi/api/helpers.py,sha256=F26nueg093a3fBYU7QxE4QXrdV3zBEOnSqLdS9AMbVg,18982 +jedi/api/interpreter.py,sha256=BQrbeTG2uKT3YtJxVria3raXqiCvYj-wl3nSxWvqOkU,2415 +jedi/api/keywords.py,sha256=kEJfvmenj2MPTs8rd5b6onI8YPjwrDo29bq5sjGmDo0,1192 +jedi/api/project.py,sha256=9s0st-ohQ6wZ-SxxstdJvF-yJZi2kXUbiFNrKvCzrpM,16543 +jedi/api/refactoring/__init__.py,sha256=8Lyv9F9Kzj5eOUF4P4xgnixIRyn7ihonqmkEgHksWLA,9579 +jedi/api/refactoring/__pycache__/__init__.cpython-312.pyc,, +jedi/api/refactoring/__pycache__/extract.cpython-312.pyc,, +jedi/api/refactoring/extract.py,sha256=u-IRD0BEVoMUDRTMRP9cD-R6DczHNsikaedOeThOBso,13933 +jedi/api/replstartup.py,sha256=o4FaVzt1PX05uHDITgu7NZkSiqMZbkdAI004b4BHy8g,950 +jedi/api/strings.py,sha256=ODTY8z9puVlQJug29pV3fufqr5N-3kixG6qxLC0Wyto,3711 +jedi/cache.py,sha256=fHEukjwaHSOiE1Z6UG5HMg-mH5fzfOTILZsWvxRaDV8,3674 +jedi/common.py,sha256=jL8TzWZ6Wl5ovayygpX5-bruk6ibu_kq5mh6FZcSGkI,668 +jedi/debug.py,sha256=WuLOoaRBKfuKEf-mZIIbMSSu2LMQJEVUS2opXQMY5aU,3504 +jedi/file_io.py,sha256=yYnkY_nUnB7pSQAjWXFyCS_8IN3BuDlXmPIuZbklPoU,2337 +jedi/inference/__init__.py,sha256=XWbXy_x_2pq9YqawkIcXFcQTNT2jlbctE8HeqWAHhlw,8505 +jedi/inference/__pycache__/__init__.cpython-312.pyc,, +jedi/inference/__pycache__/analysis.cpython-312.pyc,, +jedi/inference/__pycache__/arguments.cpython-312.pyc,, +jedi/inference/__pycache__/base_value.cpython-312.pyc,, +jedi/inference/__pycache__/cache.cpython-312.pyc,, +jedi/inference/__pycache__/context.cpython-312.pyc,, +jedi/inference/__pycache__/docstring_utils.cpython-312.pyc,, +jedi/inference/__pycache__/docstrings.cpython-312.pyc,, +jedi/inference/__pycache__/dynamic_params.cpython-312.pyc,, +jedi/inference/__pycache__/filters.cpython-312.pyc,, +jedi/inference/__pycache__/finder.cpython-312.pyc,, +jedi/inference/__pycache__/flow_analysis.cpython-312.pyc,, +jedi/inference/__pycache__/helpers.cpython-312.pyc,, +jedi/inference/__pycache__/imports.cpython-312.pyc,, +jedi/inference/__pycache__/lazy_value.cpython-312.pyc,, +jedi/inference/__pycache__/names.cpython-312.pyc,, +jedi/inference/__pycache__/param.cpython-312.pyc,, +jedi/inference/__pycache__/parser_cache.cpython-312.pyc,, +jedi/inference/__pycache__/recursion.cpython-312.pyc,, +jedi/inference/__pycache__/references.cpython-312.pyc,, +jedi/inference/__pycache__/signature.cpython-312.pyc,, +jedi/inference/__pycache__/star_args.cpython-312.pyc,, +jedi/inference/__pycache__/syntax_tree.cpython-312.pyc,, +jedi/inference/__pycache__/sys_path.cpython-312.pyc,, +jedi/inference/__pycache__/utils.cpython-312.pyc,, +jedi/inference/analysis.py,sha256=vbRF2gOLzm4lEQq9amaVEkQlWsshjl-tKEiOqUSAzYg,7763 +jedi/inference/arguments.py,sha256=zjQcIwW-m8J2Wy0MU_O_u1sTiNIXQk7du3Vcilm5FNM,12218 +jedi/inference/base_value.py,sha256=18r6-ZHXikOn2Tf6_xeC1KADz_25rTRANvAnDSvpDiA,18219 +jedi/inference/cache.py,sha256=_eOYHP3jZbLVWC2tuoBpjcLQNIWDCsDH3sCzz81mgfw,4191 +jedi/inference/compiled/__init__.py,sha256=MlKPMiQqzDJ6tUMgevROr5n0yyNubjImuRYUqM8jF1Y,2651 +jedi/inference/compiled/__pycache__/__init__.cpython-312.pyc,, +jedi/inference/compiled/__pycache__/access.cpython-312.pyc,, +jedi/inference/compiled/__pycache__/getattr_static.cpython-312.pyc,, +jedi/inference/compiled/__pycache__/mixed.cpython-312.pyc,, +jedi/inference/compiled/__pycache__/value.cpython-312.pyc,, +jedi/inference/compiled/access.py,sha256=oyiv78JZOGtRAlkjIGEEoaen6JddTQHvAIOF1GrvrSE,18914 +jedi/inference/compiled/getattr_static.py,sha256=Wv5XDvWS7Q7YtmuGp83Ey74Zhsor5BZS9597U-qRquE,3862 +jedi/inference/compiled/mixed.py,sha256=kM2bIpZX1Fvv2JPxJE3wbT1xHlvUi_PCKPr7POvZEsE,11374 +jedi/inference/compiled/subprocess/__init__.py,sha256=60v11q2VsTgmnLxsiDWOlidnoVRN2vJNE6NyE-iNYX4,19369 +jedi/inference/compiled/subprocess/__main__.py,sha256=aiYy62eWmK98om7_KvoC45eOn5fEjOHOXd0fFzVmTNQ,1187 +jedi/inference/compiled/subprocess/__pycache__/__init__.cpython-312.pyc,, +jedi/inference/compiled/subprocess/__pycache__/__main__.cpython-312.pyc,, +jedi/inference/compiled/subprocess/__pycache__/functions.cpython-312.pyc,, +jedi/inference/compiled/subprocess/functions.py,sha256=HVHbjAOwN0bgsNnVcWPsk_CLXdYy18773ueYfaCTRFM,8464 +jedi/inference/compiled/value.py,sha256=ey4FfHBfixqByy-8_ziYHKEwvEaKb60AVE5Ia4WiIlQ,21262 +jedi/inference/context.py,sha256=ioyAcWxvFLqI13VezWzCgQAAJ5NJgNn8sDcdWolxrFg,17164 +jedi/inference/docstring_utils.py,sha256=GTk3ZuWNIo8fFC9kt8NjOF_Wn0mYKjzKLrAswgvoJKI,759 +jedi/inference/docstrings.py,sha256=g5bqKJ8q7ZpLT32NVA2aCGISrG4X2aC0gDRE0OsB_IM,9824 +jedi/inference/dynamic_params.py,sha256=kjNruHLLXpJZkJ3PR0J_nRXBesxpUEzzyhqT8T8P_CE,8154 +jedi/inference/filters.py,sha256=43Dtz4_VeOF6k39YueBFvZaHj_RLC8Wtpfwt9NF28Hc,12493 +jedi/inference/finder.py,sha256=bx-TVatGVvPs696vDS-CWFbclUOdwWvXZPMJF8Vp9Gg,5326 +jedi/inference/flow_analysis.py,sha256=p-za41-mt90PvgwCZi8cCfhw1DiJCMZ2XjdCwh_rHBk,4583 +jedi/inference/gradual/__init__.py,sha256=Xzh7u-vHsvyHAzr_2kP8PY4NieFh35o-TmQY3KAttlI,143 +jedi/inference/gradual/__pycache__/__init__.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/annotation.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/base.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/conversion.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/generics.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/stub_value.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/type_var.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/typeshed.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/typing.cpython-312.pyc,, +jedi/inference/gradual/__pycache__/utils.cpython-312.pyc,, +jedi/inference/gradual/annotation.py,sha256=lkeF3gX-HWz7wQAANkCDg2gE0ccgL6VlhgDrNd4rMas,16076 +jedi/inference/gradual/base.py,sha256=g3XVBA4aX40yxpJKFI6TtOCtFrJW0HAoNECLv2xn68w,15554 +jedi/inference/gradual/conversion.py,sha256=vqVITY30kczXEuF59RV5Kp9FNx0P545ctEfCtwX9aSg,7601 +jedi/inference/gradual/generics.py,sha256=gdqzR1SlT6U-I4l6VWqRih_zSXmz69BgwcFNehXEWF4,3144 +jedi/inference/gradual/stub_value.py,sha256=PSrQqdZa-A7EtQ0EpSLrPzHWd9TQqhRwKXSVjyxCXlo,3385 +jedi/inference/gradual/type_var.py,sha256=twrYAoTRNRLHXGPs5CwOZtkvKXGGA8ynczEe9IM1jcU,4139 +jedi/inference/gradual/typeshed.py,sha256=AtR0EBoTlObIzM2c7VQI-Jn_AWsjt0YKp4NYTWOpirE,11467 +jedi/inference/gradual/typing.py,sha256=WJ9jsJN7TDz7E8jQPKVNCfzA9osIps6XyqncCnZfd3A,17255 +jedi/inference/gradual/utils.py,sha256=PXn4xXlo09fQJVfqHrhA3MdLuAn4mqqujhCTuMutzkI,1147 +jedi/inference/helpers.py,sha256=MJp-HLdtz8_jh9U62CnHVXfDZnFoyg8DlXSY1r1WdKw,5943 +jedi/inference/imports.py,sha256=9tpF3lBsrxLi1R3zEdncPNvZv_mTqhFXO-BhKpHL7VI,23082 +jedi/inference/lazy_value.py,sha256=LtD4RiWs_rQ58ZEZLp-_1Bce1B15t4GS1WpixHW891g,1667 +jedi/inference/names.py,sha256=DODbtshdoNhLgfH4AF2Asjtdl5bWamMlctoKRyYfdV0,23214 +jedi/inference/param.py,sha256=EEQZwUHY6MUjX3x4A5gl-NzKHXfJ7Ehx7Iza_s6948Q,10450 +jedi/inference/parser_cache.py,sha256=GOOahC8GvDcGW-Ct653Uts5IZ0EaymB4OwyfKm5WClg,191 +jedi/inference/recursion.py,sha256=2rSUGzs1GqMH8274KrgjLTQMLD76ld_dnopbHckM8qo,4932 +jedi/inference/references.py,sha256=60KtWGun6KFSyv8UDUEzm500jWzGosXYlxSj9GTbJG8,11407 +jedi/inference/signature.py,sha256=uh22vizTq9VsQg-5hYVPDeNs6PYV_YOW9HbCZttWo5g,4859 +jedi/inference/star_args.py,sha256=785E1MC3vvE8zQJ9w0VGei84B2lrLmvxrqYtkyehlbA,7895 +jedi/inference/syntax_tree.py,sha256=xyacV4hHY1-r5NOmL7JOZ4bvZyRgbRAaUXFgWXmmsFU,36319 +jedi/inference/sys_path.py,sha256=coTcmTe997sITzKjzXqyppFYJKpRXNO8mVaFfOIWJ7w,10218 +jedi/inference/utils.py,sha256=63DG77XBZ0b8BTKXqNl8ki-4ySyAO75IoCdFIMvttFk,2706 +jedi/inference/value/__init__.py,sha256=Pyi9BuCJvvWQue2MJH2Lsb-ffK_9MKjRVGjY5NvvhkQ,416 +jedi/inference/value/__pycache__/__init__.cpython-312.pyc,, +jedi/inference/value/__pycache__/decorator.cpython-312.pyc,, +jedi/inference/value/__pycache__/dynamic_arrays.cpython-312.pyc,, +jedi/inference/value/__pycache__/function.cpython-312.pyc,, +jedi/inference/value/__pycache__/instance.cpython-312.pyc,, +jedi/inference/value/__pycache__/iterable.cpython-312.pyc,, +jedi/inference/value/__pycache__/klass.cpython-312.pyc,, +jedi/inference/value/__pycache__/module.cpython-312.pyc,, +jedi/inference/value/__pycache__/namespace.cpython-312.pyc,, +jedi/inference/value/decorator.py,sha256=b8jWqSZQTrQ1rzThPSYCToEAC0Hfx19XEvgJsMXftus,1207 +jedi/inference/value/dynamic_arrays.py,sha256=JKezgJo4LF6gwgJ4joNp1vADX5WVhlq-5O4mhhVh1MA,7527 +jedi/inference/value/function.py,sha256=dwUEHCOwVT-lnZXaqV0P0Xrf2rPuCksXSPRVzGmrqps,17424 +jedi/inference/value/instance.py,sha256=1UthMNhM2piZaUaNuWHjOnInVKuGKqCbjdmbgCMKjyY,22511 +jedi/inference/value/iterable.py,sha256=fxMp3UiqM8xx7pHT_q30Y4thoDrBUIq62unMdBwAnpI,23305 +jedi/inference/value/klass.py,sha256=_KuA6ySY5oZgIBahdRRPOmsR5IOW8ZCwTnHdU2SLKdg,15940 +jedi/inference/value/module.py,sha256=mgmTCAOaELoMcAbmnNVbvzLIdZYXTK-vPsxA9dIOpRY,8118 +jedi/inference/value/namespace.py,sha256=yfw8pjmT705joRJSKU-CfmuiyJ-XdcelbDCzKCxqxVk,2101 +jedi/parser_utils.py,sha256=ZLs4V5LNUuMMMKrr7FHK2I7BFylM9PeRMln0wUjuOpg,11000 +jedi/plugins/__init__.py,sha256=O3yXGTvWanhnZnKubQJhsnDlyXbl23hlt2aGmhy-UUU,1445 +jedi/plugins/__pycache__/__init__.cpython-312.pyc,, +jedi/plugins/__pycache__/django.cpython-312.pyc,, +jedi/plugins/__pycache__/flask.cpython-312.pyc,, +jedi/plugins/__pycache__/pytest.cpython-312.pyc,, +jedi/plugins/__pycache__/registry.cpython-312.pyc,, +jedi/plugins/__pycache__/stdlib.cpython-312.pyc,, +jedi/plugins/django.py,sha256=n-jg8n9bXNjeuqqu46PsiyypzGjFdUjtYJIGcbiBrtI,10895 +jedi/plugins/flask.py,sha256=SRdiNzObi4Xonq3BqjdNRyz08fHPk5ZvbERyNtYgsSs,916 +jedi/plugins/pytest.py,sha256=ENS9y81QzWUpehptCqP3KVqSDftXvdpH2zzSTjQIEio,10339 +jedi/plugins/registry.py,sha256=Z3aaj2W2mpwO2Av4apD_vlkPvorJPljvy4xtJsOHrfc,307 +jedi/plugins/stdlib.py,sha256=bPNOJZ6FOrqgpVip6PpmzbJ8VlpQvxssFSCPEN_zW84,30262 +jedi/settings.py,sha256=F3EZFXS3kA6slIq2xal_qHAE8-9L3hZysKdFFLBZ9RM,3827 +jedi/third_party/django-stubs/LICENSE.txt,sha256=I16ZOWXTmaJefUk9JchiL3hxhRCIS5wFHx8YZrbzTp0,1075 +jedi/third_party/django-stubs/django-stubs/__init__.pyi,sha256=t008ZLMgegSQayAawLGBCwygFeVfDKqQEAqTpXj_9yA,432 +jedi/third_party/django-stubs/django-stubs/apps/__init__.pyi,sha256=Y3Y6Hf-auytvp3gQT19bAYTGM1e3f9BBktHVMvr2KQU,79 +jedi/third_party/django-stubs/django-stubs/apps/config.pyi,sha256=5WdAiYznWIzxBXj7DUdYwg0ZIYp2pZHi2uUuD8oumQc,834 +jedi/third_party/django-stubs/django-stubs/apps/registry.pyi,sha256=gu9N7TVSYlhW--1o8FaRhD1S7W4DbF5_tAw4VkxBcsw,2050 +jedi/third_party/django-stubs/django-stubs/conf/__init__.pyi,sha256=iD_WNEjoMcZZwVw_OsvIr-qT1D-WAyVhwGHURaVG9hQ,899 +jedi/third_party/django-stubs/django-stubs/conf/global_settings.pyi,sha256=uUXM1AzZ6y9TcZd_wEBD-MksMBnnA0S1oncpezQApoQ,17462 +jedi/third_party/django-stubs/django-stubs/conf/locale/__init__.pyi,sha256=M1UVxxBgBcKY0r7pv8DvOE6_CJavw0Od3fi8iH-APEk,62 +jedi/third_party/django-stubs/django-stubs/conf/urls/__init__.pyi,sha256=SisGWP0M2DqTBeYjxgaoseKtVjQhpvEopgidp_u0BG8,1033 +jedi/third_party/django-stubs/django-stubs/conf/urls/i18n.pyi,sha256=x4QdXccNgttRURmMrKKE5BDhfk_sc9GRyzSJPbbQDKU,297 +jedi/third_party/django-stubs/django-stubs/conf/urls/static.pyi,sha256=BXdu_MPkrP5LIynxR3lUVc6AAK_y45T8YVn_lp-UVow,172 +jedi/third_party/django-stubs/django-stubs/contrib/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/admin/__init__.pyi,sha256=EqcwNegHa1kj1bgto5_xljmuP4Ym54wt-_A7nia4WEA,881 +jedi/third_party/django-stubs/django-stubs/contrib/admin/actions.pyi,sha256=O3SXOhCX1-BkDiPxDh9BDUppLQhopWyDj2p1l71XmMo,351 +jedi/third_party/django-stubs/django-stubs/contrib/admin/apps.pyi,sha256=1q33ihcO10A_m7HB5gHdU8S1YpVohz6ihgzo9QMPYnk,142 +jedi/third_party/django-stubs/django-stubs/contrib/admin/checks.pyi,sha256=fWhQq3debqlWvigQzUZNuORAVzHw61iWmi0emeczJC8,849 +jedi/third_party/django-stubs/django-stubs/contrib/admin/decorators.pyi,sha256=Ws4zXIAH5jDgEi60FmJ_Mry9eU5rCJmpJ5lNM2XQXbs,170 +jedi/third_party/django-stubs/django-stubs/contrib/admin/filters.pyi,sha256=fQk48FO4Aik5qG-yUHr7XmW1wgJtCHM4YeWc2PEIvkU,3487 +jedi/third_party/django-stubs/django-stubs/contrib/admin/forms.pyi,sha256=vAAUpUCC7gxxUnBDCwNmPrk1L-NOCHx8lHwlceqBNDY,249 +jedi/third_party/django-stubs/django-stubs/contrib/admin/helpers.pyi,sha256=KD26oTSFhlCVWOzH2jzoAx1Oqhncx5aSUd6ySd3qR0M,4790 +jedi/third_party/django-stubs/django-stubs/contrib/admin/models.pyi,sha256=B8NtoXvvjj8pW0fyQQXwsFJ7j7j3DlaxgxuXwY4Y514,1220 +jedi/third_party/django-stubs/django-stubs/contrib/admin/options.pyi,sha256=TpeeMIIkoNO4Ky8VYdxejUzxmV8R1S3t25ofOD5Nilc,13828 +jedi/third_party/django-stubs/django-stubs/contrib/admin/sites.pyi,sha256=Di-VbTnbIqoK8xDO-IOw64s7JrRQ3CoZKYIvGXKwvOs,3288 +jedi/third_party/django-stubs/django-stubs/contrib/admin/templatetags/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/admin/templatetags/admin_list.pyi,sha256=HSelmnczfnb-E2QvjhmWkHbwTMp7zx9AoxTqWW_HkBI,2028 +jedi/third_party/django-stubs/django-stubs/contrib/admin/templatetags/admin_modify.pyi,sha256=7cJfilSXB5nWosnNDsgAMMJbZXExtCJAjJmCkvKT6a8,690 +jedi/third_party/django-stubs/django-stubs/contrib/admin/templatetags/admin_static.pyi,sha256=FV9QQ0gLLyXEtmmJqGFWRFRo-eJcvO71aRcFqZJn1sI,73 +jedi/third_party/django-stubs/django-stubs/contrib/admin/templatetags/admin_urls.pyi,sha256=BP_vzmHQ3W7lwC1qCuy7_CSL52pWF5_kyx9okRQT100,547 +jedi/third_party/django-stubs/django-stubs/contrib/admin/templatetags/base.pyi,sha256=_2N03P5IN0Y66vpFrz5l0aRyEQjJgCll8w-FJ1CsP8Y,592 +jedi/third_party/django-stubs/django-stubs/contrib/admin/templatetags/log.pyi,sha256=8m7bTnfmhLLycZoE3Aapyyz4JzO_G8I_e3L16FZATWM,454 +jedi/third_party/django-stubs/django-stubs/contrib/admin/tests.pyi,sha256=IGDA9zE9QPLNB5xgBFFr9j3AbMyufxTm2NUPqmP8mpA,1417 +jedi/third_party/django-stubs/django-stubs/contrib/admin/utils.pyi,sha256=RoMzCbcgb_dKDrY8ezXh93jNJQEEuJ3JAHDitnWZ4To,3107 +jedi/third_party/django-stubs/django-stubs/contrib/admin/views/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/admin/views/autocomplete.pyi,sha256=AmCorbocq3VfqVJnU6wtVyGESgls1a-WePP5Bl3Vi3E,349 +jedi/third_party/django-stubs/django-stubs/contrib/admin/views/decorators.pyi,sha256=xF9kl3mBijo2Ucnac0ituCn5nq2dyDYNXReYMgX-Rm8,335 +jedi/third_party/django-stubs/django-stubs/contrib/admin/views/main.pyi,sha256=MnJzYgIiYc6K7yPGAzzKokM2YwHbqYwWzVpEfQwdYf8,3390 +jedi/third_party/django-stubs/django-stubs/contrib/admin/widgets.pyi,sha256=i1bW9opYwddEL9f6hhekwMNYhg_2vbo3CPn9-DY173Y,3375 +jedi/third_party/django-stubs/django-stubs/contrib/admindocs/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/admindocs/middleware.pyi,sha256=DlifCzWN3f9RB-PoedyUCPz3MhpqODdJxEk1xRYyYu4,409 +jedi/third_party/django-stubs/django-stubs/contrib/admindocs/urls.pyi,sha256=oIadLJRRqUS4fwWe3F2TwdQViIuYuSR7iutUidncun0,59 +jedi/third_party/django-stubs/django-stubs/contrib/admindocs/utils.pyi,sha256=aKdHFqtElfEWLRx9LgJRBZby7VKeAvCFRfSh9J_yOVc,725 +jedi/third_party/django-stubs/django-stubs/contrib/admindocs/views.pyi,sha256=JvdGvkWE5fJULlrrbucwuvbOkwPSL9b2I5hsVmBZxAY,852 +jedi/third_party/django-stubs/django-stubs/contrib/auth/__init__.pyi,sha256=hDspBupLxVr58AAo69fVZa9d_pWAkORdzOcjUVxB49M,1316 +jedi/third_party/django-stubs/django-stubs/contrib/auth/admin.pyi,sha256=t3LSFWjHf93LRhNmB3Pn5bd9X4NE9kFzbYCsyDrg930,527 +jedi/third_party/django-stubs/django-stubs/contrib/auth/apps.pyi,sha256=783HiaS66kh94JjcTQDjxdhqTsecaKqZrwJ6rLSB6AQ,68 +jedi/third_party/django-stubs/django-stubs/contrib/auth/backends.pyi,sha256=OzBwXj0hB-Im6o9-2OHAOVCAkP07BG6DQzg0BIMT9d4,1605 +jedi/third_party/django-stubs/django-stubs/contrib/auth/base_user.pyi,sha256=p2mB5q8Pvv1BrE3CgnwXgfLITW3bFSNQVCp388wjxMs,1493 +jedi/third_party/django-stubs/django-stubs/contrib/auth/checks.pyi,sha256=GeMx3ueyYHAufqpprS_roDPC-MGC9jZZiiob_Yj2j1w,371 +jedi/third_party/django-stubs/django-stubs/contrib/auth/context_processors.pyi,sha256=ysQRV_JT1XA8buH6W5mkGbefSDy0nRVumXGjToqgoAg,617 +jedi/third_party/django-stubs/django-stubs/contrib/auth/decorators.pyi,sha256=D-szKzg_wbjHQsdAY7Y_SE4s7fOrE1PQRTqXDdo_acs,982 +jedi/third_party/django-stubs/django-stubs/contrib/auth/forms.pyi,sha256=jpqKdxAYIElU-TlfLzvZNefMnn1Uo3vXEJArzBainZE,3149 +jedi/third_party/django-stubs/django-stubs/contrib/auth/handlers/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/auth/handlers/modwsgi.pyi,sha256=bv6dWYpTbWYlCQjWyc76cB_lH3akN0WYDyuTmnLXiS8,204 +jedi/third_party/django-stubs/django-stubs/contrib/auth/hashers.pyi,sha256=xdR_649Y60eS-C4VcQUDn76n07U5Tzwl6DlLOcOr8Ek,1969 +jedi/third_party/django-stubs/django-stubs/contrib/auth/management/__init__.pyi,sha256=r_Nm0u9_QFypD1ZAkzWIrPzTRiKM0Vwj8N1BPgEjjUA,384 +jedi/third_party/django-stubs/django-stubs/contrib/auth/management/commands/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/auth/management/commands/changepassword.pyi,sha256=sINAEC_pucjlldxJChrjgnhv_PzIBOAi3mPOqdHYZNE,125 +jedi/third_party/django-stubs/django-stubs/contrib/auth/management/commands/createsuperuser.pyi,sha256=LI2_hmrgjeizJ0L0Pe4lo2lqvgqvfd0XDovYSbrx9bk,208 +jedi/third_party/django-stubs/django-stubs/contrib/auth/middleware.pyi,sha256=WwvDTYBlZdi21Ackk6Aksuwgt0JIOOAPOc3niDsN0J0,774 +jedi/third_party/django-stubs/django-stubs/contrib/auth/mixins.pyi,sha256=7Ca0TCSsaszq71MDnCaO2-MqySWlZc2CQQqtQZBIaL0,1150 +jedi/third_party/django-stubs/django-stubs/contrib/auth/models.pyi,sha256=nDsbZgiHmYUmC8TSu1ZAmk5uEVGlEodFN6B2dXhwaec,4641 +jedi/third_party/django-stubs/django-stubs/contrib/auth/password_validation.pyi,sha256=9KOmo5wQfvW4U_qh6yIMczt5izwIRS2GMQQ5vGX2D8I,2052 +jedi/third_party/django-stubs/django-stubs/contrib/auth/signals.pyi,sha256=0qXrIYBrx2XYwRrx1N5V5h9HgFtdmK4pzQfEMPr-EPY,120 +jedi/third_party/django-stubs/django-stubs/contrib/auth/tokens.pyi,sha256=IcHQN4Pn8ikV2qI1e4_VWkLjVbfPR5tOEM2iQR3Dz-c,361 +jedi/third_party/django-stubs/django-stubs/contrib/auth/urls.pyi,sha256=oIadLJRRqUS4fwWe3F2TwdQViIuYuSR7iutUidncun0,59 +jedi/third_party/django-stubs/django-stubs/contrib/auth/validators.pyi,sha256=3P7MK_049xYExFIZVQCaic-iuFgFGaZouQBS4RYe_6I,160 +jedi/third_party/django-stubs/django-stubs/contrib/auth/views.pyi,sha256=BPnTwECkJfap0jz0Mi1RII3PLHeHI6eH3Z3mnBZoFvQ,2478 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/admin.pyi,sha256=hQT_hpYtA2aHrknPNDLX9Fl6NSVJbdtKpWhiGOsk-VM,586 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/apps.pyi,sha256=WUN_nGxl5mnIB9IctYlkCrswHF8BpQo2yRbRywZ69c0,76 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/checks.pyi,sha256=3PeRWTpBcWB1u-66ueZ6RVWFIZReu5OOYpoN7r2JCBQ,318 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/fields.pyi,sha256=hW1z3wwu_lRFpF6Zkz2pR3pb7yh1za9PgaZFJ6o5XPY,4097 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/forms.pyi,sha256=-j9hvsPL-3syqd6O5gTcXzh3xeCQ-isFdtkhfuUwIX4,1159 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/management/__init__.pyi,sha256=B0Gku6qIM0KlOwC2zQyZVrNE2Stj4JylnzhQ4-gswiA,1350 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/management/commands/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/management/commands/remove_stale_contenttypes.pyi,sha256=-wUrCoB87Ueb8337psYunjcfoiKpxqyestpVHKzvaUE,411 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/models.pyi,sha256=PcA_Yze4dDqmxsDJqCYKm_5ZnmfaxW4VNyfGjHqkavI,1085 +jedi/third_party/django-stubs/django-stubs/contrib/contenttypes/views.pyi,sha256=wK8to6ffyMuyyrlNJu9-7nzhOASZRr-x_MWdSgd_G_4,257 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/forms.pyi,sha256=XNJrR-mHjiKpxf_r5sPxXC2De-Lyh37BL309dtR_f2I,142 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/middleware.pyi,sha256=6PzgiwFmjk4K4EBW7tfiUtdxI0x-f81N6RmbMOa5e1g,299 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/models.pyi,sha256=sd9sLh0PBc86GpcJrEtE4vaPVvs8xE1gifekJbBuyDo,445 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/sitemaps.pyi,sha256=-JeEn0-5EwpyJHOtgYd8IK_C8fGTmLPtk_UpgiwsiX4,81 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/templatetags/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/templatetags/flatpages.pyi,sha256=U1qIuNadeeNP9oodmb1pXDt9Mrc7urXsAvA0c0bLmC0,518 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/urls.pyi,sha256=oIadLJRRqUS4fwWe3F2TwdQViIuYuSR7iutUidncun0,59 +jedi/third_party/django-stubs/django-stubs/contrib/flatpages/views.pyi,sha256=aJzjM06Iu8BG4w5frT51cpHotQ6e4GKH18Cukc-yDbo,315 +jedi/third_party/django-stubs/django-stubs/contrib/gis/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/gis/db/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/gis/db/models/__init__.pyi,sha256=I8SirL0ClZODX-As42cycJ4VjKnCc97cK9hVKBY6hH0,418 +jedi/third_party/django-stubs/django-stubs/contrib/gis/db/models/fields.pyi,sha256=HgR0BvodeWJ2iGkP8pLuFvI9SQ5DTWbP0oMzsZwEbHU,3133 +jedi/third_party/django-stubs/django-stubs/contrib/humanize/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/humanize/templatetags/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/humanize/templatetags/humanize.pyi,sha256=4pEElOr5jWoM1AAc-wq4HSLz-0iSrGwmeqqlJqdq2fs,619 +jedi/third_party/django-stubs/django-stubs/contrib/messages/__init__.pyi,sha256=UHr4xfoH6faNXSh6Xq-6OaRhgsi57MkDktuW_MgfDDs,524 +jedi/third_party/django-stubs/django-stubs/contrib/messages/api.pyi,sha256=ue6Dc8C-dHtJuRlcoQE5zPPsmSGOkgUem_H35681jk0,1169 +jedi/third_party/django-stubs/django-stubs/contrib/messages/constants.pyi,sha256=gBM2s_ImbX0K4W0mcwvmesgrEboAd_f_Pzq-lbUQ-IU,187 +jedi/third_party/django-stubs/django-stubs/contrib/messages/context_processors.pyi,sha256=FSJ6CYdzc9OLaBZyMQe1UlTIYq7ROchz_wBsQn-tEGo,249 +jedi/third_party/django-stubs/django-stubs/contrib/messages/middleware.pyi,sha256=H237yhPKbsl-Civ4R6Et_LX5uBUBLrDqTDvBT1JMef4,349 +jedi/third_party/django-stubs/django-stubs/contrib/messages/storage/__init__.pyi,sha256=ySIlvAEkKK1c5eDrg4u2cs-Mu-xkBuIjVqMjlOMqNBo,202 +jedi/third_party/django-stubs/django-stubs/contrib/messages/storage/base.pyi,sha256=EWpGkx7m2iNNqXTYTYTgeQtTmdH_loN6ba3XHbZo8QU,907 +jedi/third_party/django-stubs/django-stubs/contrib/messages/storage/cookie.pyi,sha256=h99T_FYt4AvNx7MaYtCJ42yuXxw2-u4dDaWgX2aqOHQ,487 +jedi/third_party/django-stubs/django-stubs/contrib/messages/storage/fallback.pyi,sha256=9FSrcnvxbwiGz34cVG8v8jOPOBl7cXK8Nvd97wbUCVs,240 +jedi/third_party/django-stubs/django-stubs/contrib/messages/storage/session.pyi,sha256=MsQOXu0WRhdg3mjL51hB_VYTSYspwgCyUfwK7pyEgyA,482 +jedi/third_party/django-stubs/django-stubs/contrib/messages/utils.pyi,sha256=BMBEulwgQtm_k9l8rf5DNeZ-KpKvth7kqf7kzEL2908,69 +jedi/third_party/django-stubs/django-stubs/contrib/messages/views.pyi,sha256=kkAB26i4kilkRl9AJbWCbW5zoy_uuKr33MBLu8Fp4qA,308 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/aggregates/__init__.pyi,sha256=dDW1jG5QW0rYRWbzlmBQ8PT2v9ZdWgFJ0IRHhvBJQyQ,540 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/aggregates/general.pyi,sha256=s1jdffCAM8P8TUYWwwu5ZAjm38N8K7wU_UWR_2wR2SI,338 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/aggregates/mixins.pyi,sha256=BE72JG-si9GbSvdXWBTN2asV9jYrZIh2GldTVScgC1I,29 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/aggregates/statistics.pyi,sha256=PqsAq9E7Jf6IUi3xYpm9LMCl52R6XMfMQieQxe-d49w,470 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/constraints.pyi,sha256=s2FgJy9Rl1JnijuF4VQKTz6aVahFoJAwnO1C36sJQ4I,576 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/fields/__init__.pyi,sha256=nAMbC_GxM8b8-dR9AlAQ9W__vybgo-3-X6bSBi0un48,695 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/fields/array.pyi,sha256=B19PlhiKI8jhgJQq6fK_2SMXtZ1-yq0RyGe6QWwPHik,1729 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/fields/citext.pyi,sha256=bC3-GS1JBsGcBiF5ve1dN3O8vG2eKjJSH0UB7mO7kFA,216 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/fields/hstore.pyi,sha256=f6qGgeaD_ypIuIsPMGlgm_D48zms5vJ5VCclCp-iSCc,519 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/fields/jsonb.pyi,sha256=XQpN-W-SzUoLm8M84ebg9ccGTZ51oMvpNU2S7PGdYMo,1011 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/fields/mixins.pyi,sha256=NAiNjruhDqjVpZdusWemB8_eZhe54CHtvAGKDjvHFKY,113 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/fields/ranges.pyi,sha256=yq6_zMfmqvqbCO627VtFM96DiGfUgdp3tku9RBxv7Vo,1342 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/functions.pyi,sha256=__zcyTUVl9N7_5NVTzZ_SAXvflnckIVZtz46hcSiesI,95 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/indexes.pyi,sha256=WsAYW3VLoP4rQ4p8ra4gkNPYcBH4XU6R-5b5HfvRZ8I,2272 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/lookups.pyi,sha256=6XUZ02CEjDiE_s6sGMCTHuLk1R_MtdkGGjAASJzGVEk,579 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/operations.pyi,sha256=Z8SEkq9xL8vu8ZhaEUZr9DqaN-oBSeu855ng3KXSGXg,735 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/search.pyi,sha256=m0sqtO97PEv8__bspxNgeLfBaMga2rnZUnkHl-6jckk,2195 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/signals.pyi,sha256=yeuM0ob6OaYoACmfLBdZCNWgPlm7A9ZwWjoZi-7Kb0M,237 +jedi/third_party/django-stubs/django-stubs/contrib/postgres/validators.pyi,sha256=s2eEtAs0EGPZdPztDX4Q4ZLLf5fm23Z_ggZU4yWSo0k,653 +jedi/third_party/django-stubs/django-stubs/contrib/redirects/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/redirects/middleware.pyi,sha256=fKa3O8n_i5HyNVUHXOaTIar7d5SmNOPwUqmo5FRDqx0,391 +jedi/third_party/django-stubs/django-stubs/contrib/redirects/models.pyi,sha256=h-aCkCB9-5rhIRn1zd1DhgCRzFbwbi8n3ogO2THqDDI,168 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/backends/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/backends/base.pyi,sha256=Ny7D5lAHrEndxAQ_1CPFZZM2DsHMoMUnBdJanHOSw0k,1527 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/backends/cache.pyi,sha256=7wCasZDS2jUxauFRfc1hgCbbMhdu3BV0eyNaJDMTTnU,299 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/backends/cached_db.pyi,sha256=8jDkOaweqsMtw0xxE-TMfqKVF-EOhZYEN3kjasw_f9w,305 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/backends/db.pyi,sha256=tVGmHGKuGZGDYi4l8aR1a6BlQKRYYDrNI1qorOiCOYI,577 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/backends/file.pyi,sha256=cT7G4hn-3A_ZdlWH1TPH8K_Q9bAoKUUjjLlAwFvFPro,283 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/backends/signed_cookies.pyi,sha256=qBbI8hMn_5IEPkzq2AQW54pFZN-d0lVUhSZ0W6DgC84,100 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/base_session.pyi,sha256=1NPvIW3qfHt-WF1wOYh7FGLoPasYJ80KTw0pdrWOh3c,665 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/exceptions.pyi,sha256=JjF0Gti8-WnILz7dVoTAv1yCrbhMLsvBe3e1z3Zzu50,156 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/management/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/management/commands/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/management/commands/clearsessions.pyi,sha256=Gk-hDddr6HHr5PArycz3DqoeF476UpGqav9HGp_NsnI,85 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/middleware.pyi,sha256=ZaLlZVRmZZgZaDMmC-j4YZVKDSMjdFYW7WOTru56gOg,478 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/models.pyi,sha256=5Xexgn7rtazccT-DG_4gm4-Lc1NtQTZZBYZQn9LdWdk,176 +jedi/third_party/django-stubs/django-stubs/contrib/sessions/serializers.pyi,sha256=IDMgSeFmLRJYNp5_5jiYaTPKxPBLaEVbEWqPbSqRYaM,311 +jedi/third_party/django-stubs/django-stubs/contrib/sitemaps/__init__.pyi,sha256=jH3-BaWLaKwI5DATtZlXtMLNVETNStRqeviv6zqvv4M,1559 +jedi/third_party/django-stubs/django-stubs/contrib/sitemaps/management/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/sitemaps/management/commands/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/sitemaps/management/commands/ping_google.pyi,sha256=Gk-hDddr6HHr5PArycz3DqoeF476UpGqav9HGp_NsnI,85 +jedi/third_party/django-stubs/django-stubs/contrib/sitemaps/views.pyi,sha256=JueiGm9sG9TSRfC_6FP7SpdjeOxKko1he1IJJmvWl2Q,762 +jedi/third_party/django-stubs/django-stubs/contrib/sites/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/sites/apps.pyi,sha256=hA4WuIXyL9iDiIYzysxfk73L--Va48Ede2GGIvVCxPQ,69 +jedi/third_party/django-stubs/django-stubs/contrib/sites/management.pyi,sha256=tMS5XtfebpRS5cGVshTDsfF5tQmiBZqdVLH3Exey-vw,288 +jedi/third_party/django-stubs/django-stubs/contrib/sites/managers.pyi,sha256=9NB542AluxPiwc0ey-I-FDQKSlria3-A9ROMwdg6GXI,170 +jedi/third_party/django-stubs/django-stubs/contrib/sites/middleware.pyi,sha256=XdobZMJ16FQk-rt57wCtkkJ2NoJfD1LQrRz1lZfx8ZA,209 +jedi/third_party/django-stubs/django-stubs/contrib/sites/models.pyi,sha256=CAVo7Xo9pU0SQQ_-CBooqLbMaUjgqugtqvQUXAcmLrM,614 +jedi/third_party/django-stubs/django-stubs/contrib/sites/requests.pyi,sha256=9cA6iRYob33A_N7AHCWhB5HV0u7CBaa3YFxBwKtNjXc,298 +jedi/third_party/django-stubs/django-stubs/contrib/sites/shortcuts.pyi,sha256=dPbrudBzry6CawfRV1IbbF7SHGvSa6oDjGJayELnFzI,266 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/apps.pyi,sha256=JiBc5TPKiv6yGihDg8hKxxONGJOnOn-rZmbU_P4rbxc,126 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/checks.pyi,sha256=_LvpylIL3RjCRsZOzCn7MMuBD41DFLEZNWDolU61Ihs,242 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/finders.pyi,sha256=uYtF_5sLYizgcESyiWVSsqwCHonIPf2CPF1Lh0Qg09w,1715 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/handlers.pyi,sha256=H6xAYLSqzoYTmTD20s-nvsaEaWChXbQWcMsllwq5zJQ,417 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/management/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/management/commands/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/management/commands/collectstatic.pyi,sha256=aauFD0KDUGYrHRVy-El3hUVhCkTXtWty213H0YRRjHc,1125 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/management/commands/findstatic.pyi,sha256=AoPCqwNz4yYQQm1z6WCDMdxcWrBURx1I5iPsHJQvdOo,87 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/management/commands/runserver.pyi,sha256=3uTSyH43HWzNj2ysIJXyT32MaF4q1_gz6q2hJJy2nRM,136 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/storage.pyi,sha256=WRfGAaDXcvPjl7ICgAkaxm_lD8dNx9EqikVnBd-Bie0,2262 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/templatetags/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/templatetags/staticfiles.pyi,sha256=3rNdwJS7hrGHEmWcCDQXI195iq8q2A_fB9K-mjIFLtg,234 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/testing.pyi,sha256=sD4CDNL4bD-9nTfvLSMK8D7_Oc0UlxpMgraoUAGzr7s,100 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/urls.pyi,sha256=e3hDRnM55Tey8gm6uIZkU9XaKL6K1hQCqbgwo2JEfD8,198 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/utils.pyi,sha256=5kh8wIGL3tKBTFQ2tv-beeXeSlQvKS9_S8LVXflcybg,438 +jedi/third_party/django-stubs/django-stubs/contrib/staticfiles/views.pyi,sha256=Q2CX7w--nytsntQ6IkUxHuthpEHo8HQMeIsaC4Moq1c,222 +jedi/third_party/django-stubs/django-stubs/contrib/syndication/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/contrib/syndication/views.pyi,sha256=1HnKaVl09EenWI_V6xo-qcobr45hCSLC3uUtoimfGoU,1251 +jedi/third_party/django-stubs/django-stubs/core/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/core/cache/__init__.pyi,sha256=6zFjJ1q_xONqYBqBOcphnTqp49ZU8UrglfT-DCibJ-A,717 +jedi/third_party/django-stubs/django-stubs/core/cache/backends/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/core/cache/backends/base.pyi,sha256=YQGqJhft7pHV6PSlOwIi5WMK_Zhpqld19F1zp6axjnI,2327 +jedi/third_party/django-stubs/django-stubs/core/cache/backends/db.pyi,sha256=wGdUsoR-ow-u2LRIjaakjcwPByKQb8-U-KOXhTbW9wg,595 +jedi/third_party/django-stubs/django-stubs/core/cache/backends/dummy.pyi,sha256=PilEF_Ep1-fDe4bQFkVaFm_n4uTLtmzHv2ewWM8VBzo,182 +jedi/third_party/django-stubs/django-stubs/core/cache/backends/filebased.pyi,sha256=-ShvV4mAtuzKC617Q2qMr7FHZkwoBpG_KP-pfOBW02Y,216 +jedi/third_party/django-stubs/django-stubs/core/cache/backends/locmem.pyi,sha256=kz1dYtPcAvhw-OAgRG0Vd0n92M_1xxN8iaGoU-s4FZA,186 +jedi/third_party/django-stubs/django-stubs/core/cache/backends/memcached.pyi,sha256=08qEPXnng3yPOK9ZYP4WtWrtENsavVyQbI0ebWLffjI,352 +jedi/third_party/django-stubs/django-stubs/core/cache/utils.pyi,sha256=2rG3g9PW3FWNflGxI84bpIfvpVQlzYYg7FaZre1vEA0,184 +jedi/third_party/django-stubs/django-stubs/core/checks/__init__.pyi,sha256=lNFmiy3qR0A9eHFRAQT7h5sP7a8jHqRUDc6RhhBnjRo,430 +jedi/third_party/django-stubs/django-stubs/core/checks/caches.pyi,sha256=ILPwDMFMEiAA7KYy_G2He5djgU5iZiA8-kq3t8jgeTM,267 +jedi/third_party/django-stubs/django-stubs/core/checks/database.pyi,sha256=a86fSgBJq1RN7ReWxynzDzEJeRf-_SNW2pP5RuavEL4,103 +jedi/third_party/django-stubs/django-stubs/core/checks/messages.pyi,sha256=crXlAhQmqdtkL65DgmJf7T1h7zPJ6LlqXq2wVH0Nx1w,925 +jedi/third_party/django-stubs/django-stubs/core/checks/model_checks.pyi,sha256=YUzLAMcJQ8wgRiN41JJ1kUlkArL9cNW-mzN4z4qIPQ0,358 +jedi/third_party/django-stubs/django-stubs/core/checks/registry.pyi,sha256=djmQNgx4H932hka9_kArUfRrH-MRxMfQS5SchBapFqM,1212 +jedi/third_party/django-stubs/django-stubs/core/checks/security/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/core/checks/security/base.pyi,sha256=iPHQ9r4dT1LPGKvNGYzbup-1So13CsG4CB88tcL0s5w,1592 +jedi/third_party/django-stubs/django-stubs/core/checks/security/csrf.pyi,sha256=ql6NePz-3tHvFrRs1-RiUL7hu-YTeqvOjJDJKWECx4o,379 +jedi/third_party/django-stubs/django-stubs/core/checks/security/sessions.pyi,sha256=GtTSJmziKaY-VxLKo9uZ2Iv2Bm_Q1GucnscQ9IcqI3k,527 +jedi/third_party/django-stubs/django-stubs/core/checks/templates.pyi,sha256=onbDwInTuNoww68Ei0NcAl3zBYRJ1YXTL7ihxAhfyN4,391 +jedi/third_party/django-stubs/django-stubs/core/checks/translation.pyi,sha256=KB0XkeUqlgS1O-HFeVcF0gP09nv9su--tX6fJmTNFKA,155 +jedi/third_party/django-stubs/django-stubs/core/checks/urls.pyi,sha256=cPGPCVBb1TtYLLIgid6su5NcNd9p90b7-40uT9kYhq4,780 +jedi/third_party/django-stubs/django-stubs/core/exceptions.pyi,sha256=VPgD4WzafSFza0Gv5-D0WmJrdtcBAkBKhLirWk9n4h0,1543 +jedi/third_party/django-stubs/django-stubs/core/files/__init__.pyi,sha256=hdnkFb2OQ7nFCvlNdWdVo4qsMPLXN_YaeyRdMAineH0,68 +jedi/third_party/django-stubs/django-stubs/core/files/base.pyi,sha256=EEh_-8b3WGLzwLAKVaKHe_-FNW4yipUv4cEWCThurls,1435 +jedi/third_party/django-stubs/django-stubs/core/files/images.pyi,sha256=SMT0wolMYso4QmCGlPtKmBirJOnvghPc8TkUgMFGe6M,309 +jedi/third_party/django-stubs/django-stubs/core/files/locks.pyi,sha256=EHZ7epp-o3EYCF3vCAPw3DLI5QAPNTg63LDLJL-BP50,308 +jedi/third_party/django-stubs/django-stubs/core/files/move.pyi,sha256=PISpZOttN4V_udjfb3adCtD-4oZ6Xaq8W79I6MddcrY,130 +jedi/third_party/django-stubs/django-stubs/core/files/storage.pyi,sha256=JeqnJwGpqWlD-FDfKMVYM0uTcoHcCaFehqwxyrFcOsQ,1769 +jedi/third_party/django-stubs/django-stubs/core/files/temp.pyi,sha256=GXys-R-G1z3HKO90VbMZqW5mpy8e468QbczNs9k9vQM,100 +jedi/third_party/django-stubs/django-stubs/core/files/uploadedfile.pyi,sha256=mmdboWz3QcaLVWHtiB4GWI5GowWOgQXPpSZoZGJuxqo,1486 +jedi/third_party/django-stubs/django-stubs/core/files/uploadhandler.pyi,sha256=yDWGn8nxgymZGsTNoPles2N1JgzZfvJnR8SGQivSRA8,3195 +jedi/third_party/django-stubs/django-stubs/core/files/utils.pyi,sha256=-YkCeClNFbCHxU418Pbn4t8p451gwLt5UfHnv-1twCc,547 +jedi/third_party/django-stubs/django-stubs/core/handlers/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/core/handlers/base.pyi,sha256=zr5dcNLmsLL8WK45_pof5Whiw8JyVqNe-nQNNTaRBUc,572 +jedi/third_party/django-stubs/django-stubs/core/handlers/exception.pyi,sha256=9Z2sl79zEx6EkjxA7930oIYYsuaOZQNKf_zNUa21YV0,570 +jedi/third_party/django-stubs/django-stubs/core/handlers/wsgi.pyi,sha256=ZmdUpvNrkmvBNxddhxasurnqfL2cXAYWiDchJMTFfVw,1325 +jedi/third_party/django-stubs/django-stubs/core/mail/__init__.pyi,sha256=fS7DQB-KYK0_XQ5JJ_mfNooDw4aI8K3DQOGmUW2cjKQ,1504 +jedi/third_party/django-stubs/django-stubs/core/mail/backends/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/core/mail/backends/base.pyi,sha256=gRQflStnEnexigKiEhXW9ew01NFWHkgEErsO7zGkmq8,604 +jedi/third_party/django-stubs/django-stubs/core/mail/backends/console.pyi,sha256=rp3gJ_WRrP7cC6OHCZxDmMCEGpwSZTXTE_-9sYGE7qA,103 +jedi/third_party/django-stubs/django-stubs/core/mail/backends/dummy.pyi,sha256=rp3gJ_WRrP7cC6OHCZxDmMCEGpwSZTXTE_-9sYGE7qA,103 +jedi/third_party/django-stubs/django-stubs/core/mail/backends/filebased.pyi,sha256=rp3gJ_WRrP7cC6OHCZxDmMCEGpwSZTXTE_-9sYGE7qA,103 +jedi/third_party/django-stubs/django-stubs/core/mail/backends/locmem.pyi,sha256=rp3gJ_WRrP7cC6OHCZxDmMCEGpwSZTXTE_-9sYGE7qA,103 +jedi/third_party/django-stubs/django-stubs/core/mail/backends/smtp.pyi,sha256=9fsENrJbPIEXUtnYhW_1dADqpcj7I4Z3kR2-3DGwdHg,510 +jedi/third_party/django-stubs/django-stubs/core/mail/message.pyi,sha256=AJVmTc2Ih1J4hsxLyE5ESK8r611Ooqk1thh39cW28v0,4094 +jedi/third_party/django-stubs/django-stubs/core/mail/utils.pyi,sha256=HiKGzyDfshoDMKu0_OuSD262A10HLjhURdD4ZNJt1ec,95 +jedi/third_party/django-stubs/django-stubs/core/management/__init__.pyi,sha256=xqh8MGApVKxQeURNp9Rp8zkm_4VlnefNkcDNnLiF6Yw,841 +jedi/third_party/django-stubs/django-stubs/core/management/base.pyi,sha256=ek1zS9nL1nCODxJGB2tJ6UXz2KjHoJzXeYa54B_wJU4,2922 +jedi/third_party/django-stubs/django-stubs/core/management/color.pyi,sha256=vnwktv_gcMVYi1TUqZBJ1DQZzpUMkwEzyZxs4FE47OQ,1052 +jedi/third_party/django-stubs/django-stubs/core/management/commands/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/core/management/commands/dumpdata.pyi,sha256=BnOg4pY4rHhpAptNOtprYlnqdHCdnFygXlshZu_g62Y,123 +jedi/third_party/django-stubs/django-stubs/core/management/commands/loaddata.pyi,sha256=jbmimF-dEcZb-DftHd6JuveAc_o50YWVGpnajeiVKOo,625 +jedi/third_party/django-stubs/django-stubs/core/management/commands/makemessages.pyi,sha256=hXQnwklaQn5pUiJSvIq757ECX5yJd5IcE3X6-d78G84,1134 +jedi/third_party/django-stubs/django-stubs/core/management/commands/runserver.pyi,sha256=if7vEdyAi4nKamysFhqo6H_JDGRhSPSZhEGigVamK4g,194 +jedi/third_party/django-stubs/django-stubs/core/management/commands/testserver.pyi,sha256=Gk-hDddr6HHr5PArycz3DqoeF476UpGqav9HGp_NsnI,85 +jedi/third_party/django-stubs/django-stubs/core/management/sql.pyi,sha256=fi7T3XJ92IYa50gpmW8vwmA9QPag-oU5XLP2WBo6rpI,429 +jedi/third_party/django-stubs/django-stubs/core/management/templates.pyi,sha256=vHW_QxpqF7tt5xpREpcMA7pdHKkY5E53MkzQaTtkASQ,624 +jedi/third_party/django-stubs/django-stubs/core/management/utils.pyi,sha256=UfBxh4njLLSBSor3WdNJt1clcHylFcqwkiSivhLHK3s,527 +jedi/third_party/django-stubs/django-stubs/core/paginator.pyi,sha256=3ha_zAANkWUw_GUkgCYaQULZQRGcxXCS5gaBdkozVdg,1896 +jedi/third_party/django-stubs/django-stubs/core/serializers/__init__.pyi,sha256=fLovD32hc1-1o1kK0wjYy7LdtzF5VcQ0C-Pc8Ta1wGo,1377 +jedi/third_party/django-stubs/django-stubs/core/serializers/base.pyi,sha256=5Lcy5QIKxdt26oPFUeZS6ix1WGk7UlsFLnYLFCgM-GE,3388 +jedi/third_party/django-stubs/django-stubs/core/serializers/json.pyi,sha256=mAtdKicHMrrrYDSb_3TUv5lhhg41Rbdukqrupkc8SqI,422 +jedi/third_party/django-stubs/django-stubs/core/serializers/python.pyi,sha256=QctI1p3qqQiaqLpGJdLwvIvECRDz5taXFR1ElFZB1x0,535 +jedi/third_party/django-stubs/django-stubs/core/servers/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/core/servers/basehttp.pyi,sha256=7PZLnV2hBprVabl_Ph9E6v0w4pjdXcbWKP49uijiD1A,1301 +jedi/third_party/django-stubs/django-stubs/core/signals.pyi,sha256=oSxvwX4-JFULZ6XVH3IH9fIoB7yQP7qWf1q2ifN1wNw,163 +jedi/third_party/django-stubs/django-stubs/core/signing.pyi,sha256=nMLdl0czCgg1pQPSO-8Y8DRyUlbk6t_czM4GzRe1pAA,1467 +jedi/third_party/django-stubs/django-stubs/core/validators.pyi,sha256=EeHJ3xcX9cpZ75vv8vVgZ7XdrRXHU0x8c40lJYMq9G0,3934 +jedi/third_party/django-stubs/django-stubs/core/wsgi.pyi,sha256=6VtHyJCF0P4EmH0PppqXcZMZ3kV75FuSjKglcvueZ9M,98 +jedi/third_party/django-stubs/django-stubs/db/__init__.pyi,sha256=RFbCnbvJwrVkhhkwk-OUM4Q7hCIx88ESSptH3qqQGhE,936 +jedi/third_party/django-stubs/django-stubs/db/backends/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/db/backends/base/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/db/backends/base/base.pyi,sha256=NvNs0Fpr2hOcUhZu0WswbNUaxheO1_haci7_WcMnFgs,4348 +jedi/third_party/django-stubs/django-stubs/db/backends/base/client.pyi,sha256=M2_0vbL99q0t6Y0mKHT-TJ0FHOnXrfz0UFVxQxHIqWQ,274 +jedi/third_party/django-stubs/django-stubs/db/backends/base/creation.pyi,sha256=x9cz11-3RbcDHEAmN2Gr6Efvg3wMqhMx1tweI_z8jbU,1103 +jedi/third_party/django-stubs/django-stubs/db/backends/base/features.pyi,sha256=cbWlLXRFXtvJTHSOm71kTMmvlYLvzVejJ6PIu4_R5PM,4244 +jedi/third_party/django-stubs/django-stubs/db/backends/base/introspection.pyi,sha256=CGlvGS9dFb32sBC_llaLmhVlBxwcO26JTsdA3mW8-9A,1490 +jedi/third_party/django-stubs/django-stubs/db/backends/base/operations.pyi,sha256=-k3Ca4u2vUCVTYS04vxgzD-XpX7PCE9o6RBP_--kJKs,6237 +jedi/third_party/django-stubs/django-stubs/db/backends/base/schema.pyi,sha256=PrP7x6JM2lfZFJcZYIXN7OjevwSeJXHX77uIkFFGcv8,3346 +jedi/third_party/django-stubs/django-stubs/db/backends/base/validation.pyi,sha256=c-X0DuxTKiXBUrZ1xrN2vxE9EC0hRfG2aS3tg3ikZyI,386 +jedi/third_party/django-stubs/django-stubs/db/backends/ddl_references.pyi,sha256=WsOtVIe0GMwG-3kuNxjJaAjg7NDl0FgPfKAIufsvIDY,2657 +jedi/third_party/django-stubs/django-stubs/db/backends/dummy/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/db/backends/dummy/base.pyi,sha256=Bybortoqc2aftoDUI9JOW6B3eA8RmtNnPI8IswCBw_w,1029 +jedi/third_party/django-stubs/django-stubs/db/backends/mysql/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/db/backends/mysql/client.pyi,sha256=LuRBeD30YTt7ZKkIy0QPI5cUoCdDkcFUphPF3P6VHmU,383 +jedi/third_party/django-stubs/django-stubs/db/backends/postgresql/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/db/backends/postgresql/base.pyi,sha256=m4ZIq3xup97EwKqz215v0DPzkl64DGJ8ckiJOzWkPHg,459 +jedi/third_party/django-stubs/django-stubs/db/backends/postgresql/client.pyi,sha256=KnnW4cVB55-O1rySn8y0wxf1HmInCvUppc4Ylzg1HVM,281 +jedi/third_party/django-stubs/django-stubs/db/backends/postgresql/creation.pyi,sha256=fnJLgXWLVw9MDbkPM_PrlgCNeEovyRFWahmCpmGtL9A,117 +jedi/third_party/django-stubs/django-stubs/db/backends/postgresql/operations.pyi,sha256=8l4IrNRrQZJk-LLbH3zGWqgpTmBXA2-rUsP76GgpY_Y,125 +jedi/third_party/django-stubs/django-stubs/db/backends/signals.pyi,sha256=fLFzAfAJ6fuAuy0wVTa0zHBeFCgG7bMLOi_kT7diVBs,69 +jedi/third_party/django-stubs/django-stubs/db/backends/sqlite3/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/db/backends/sqlite3/base.pyi,sha256=bVWwqWyV2s6IXlN9-njHCzdAwLlhw05GFlNq9QT-xG0,349 +jedi/third_party/django-stubs/django-stubs/db/backends/sqlite3/creation.pyi,sha256=fnJLgXWLVw9MDbkPM_PrlgCNeEovyRFWahmCpmGtL9A,117 +jedi/third_party/django-stubs/django-stubs/db/backends/sqlite3/features.pyi,sha256=QBNQPU-dQwPzmhxGH3AcxhgcJ5rCwtLeWJbelW3Lw3E,117 +jedi/third_party/django-stubs/django-stubs/db/backends/sqlite3/introspection.pyi,sha256=7bAlv3kwuzQxii9IDzIOLFHKsr2_TjCkC9wBFmEB2CQ,363 +jedi/third_party/django-stubs/django-stubs/db/backends/sqlite3/operations.pyi,sha256=8l4IrNRrQZJk-LLbH3zGWqgpTmBXA2-rUsP76GgpY_Y,125 +jedi/third_party/django-stubs/django-stubs/db/backends/sqlite3/schema.pyi,sha256=LYEtIfhLNap5vTN1wwBi2NvjgvyRQPhbFpMAYOybr68,127 +jedi/third_party/django-stubs/django-stubs/db/backends/utils.pyi,sha256=yDvBVaKmX_paln-6-vFqr6EQt-vqucdEqkZm3UKvpnA,1835 +jedi/third_party/django-stubs/django-stubs/db/migrations/__init__.pyi,sha256=hhLMlg4osD2PgLKlq76pVHYWqPXZMnA4BpE9v8tDWyQ,243 +jedi/third_party/django-stubs/django-stubs/db/migrations/autodetector.pyi,sha256=xQmcpsLs8jTPIMGuwaASncuUExFQeutQDKJdzNwjAQE,3011 +jedi/third_party/django-stubs/django-stubs/db/migrations/exceptions.pyi,sha256=qoLgU_vrG4a68pzirMdWXBunpPEbYOxdV_m7hJrfKh4,709 +jedi/third_party/django-stubs/django-stubs/db/migrations/executor.pyi,sha256=0s4j-NtGdXyt6dN-GUzjQfPmvvuZAiO-qph4OSIk0_I,1679 +jedi/third_party/django-stubs/django-stubs/db/migrations/graph.pyi,sha256=gZ2DtdRW8WGaNagYSS450xy37jIEYHwXrp4NK6UEUfo,2531 +jedi/third_party/django-stubs/django-stubs/db/migrations/loader.pyi,sha256=FzXHEFMvG28529zVgYPqvIUVmgu4b6ItJHmyLDOcjsU,1547 +jedi/third_party/django-stubs/django-stubs/db/migrations/migration.pyi,sha256=EC3sKQH__KdAQHdtVlB9Mdng9PZRjXfRr5kq_C4QIsU,1064 +jedi/third_party/django-stubs/django-stubs/db/migrations/operations/__init__.pyi,sha256=7V9NE9LUjCDMmYRbbA_DpyByGdwqYJGP4PP4eb9irGM,796 +jedi/third_party/django-stubs/django-stubs/db/migrations/operations/base.pyi,sha256=Yf5Kls_4hSQsIXjlL9OK08vwdBoC-U9HZUEqFAgt3iU,902 +jedi/third_party/django-stubs/django-stubs/db/migrations/operations/fields.pyi,sha256=K3iNfmrU5ih3R9q5iWbWd9vzvwfdKEI5t1mTSOQp6PM,1166 +jedi/third_party/django-stubs/django-stubs/db/migrations/operations/models.pyi,sha256=-PMXj0mF4RpqFF5EfGbXLjb6OL6M3N-IiLppkp1zhNU,3154 +jedi/third_party/django-stubs/django-stubs/db/migrations/operations/special.pyi,sha256=IUylNYPs2amdxe0LXDpAcgG38E6MidF-HqfSXB-rqT4,1340 +jedi/third_party/django-stubs/django-stubs/db/migrations/operations/utils.pyi,sha256=c3JLPgnNvLkXPqd58STsQgONi7i1eUn081TgQaGOehY,217 +jedi/third_party/django-stubs/django-stubs/db/migrations/optimizer.pyi,sha256=4oDcfA4PXyGPtc1C6MmknB-6TaaNtYdeWSyE7JTmMzk,343 +jedi/third_party/django-stubs/django-stubs/db/migrations/questioner.pyi,sha256=734ewVOcc5I9wlAUW2DJ8BVBqACx6C1pm4cHY0ZlwUo,1156 +jedi/third_party/django-stubs/django-stubs/db/migrations/recorder.pyi,sha256=j2th14mIxchOJ3s1ar2nbloAI6Xy4wiVDitnw-i9snE,806 +jedi/third_party/django-stubs/django-stubs/db/migrations/serializer.pyi,sha256=GRls6awL8QgjOVzgNassbBe6H4bEhT1h1vgdjxD97T8,1828 +jedi/third_party/django-stubs/django-stubs/db/migrations/state.pyi,sha256=3Hflqo8oiCVz-oSprFYyKetkhQsjU69QtTViZimP5Q0,2640 +jedi/third_party/django-stubs/django-stubs/db/migrations/topological_sort.pyi,sha256=NJzfRJYQwJeJS8fCVme_hYGsbrt14h7EGAcs2W0fmL8,348 +jedi/third_party/django-stubs/django-stubs/db/migrations/utils.pyi,sha256=yRlE8__8quhMrlAMU1F282hQftrxxiSQbja20o8hYmc,207 +jedi/third_party/django-stubs/django-stubs/db/migrations/writer.pyi,sha256=cNq1r8wj7KawULnICxXBTWDqO0QVGCoBKLls3O_o48M,1441 +jedi/third_party/django-stubs/django-stubs/db/models/__init__.pyi,sha256=7nR-mr7WV8G0f6Cs5Hcblfww2y2WzC1oQxd31mhnx0Q,3576 +jedi/third_party/django-stubs/django-stubs/db/models/aggregates.pyi,sha256=PrOHocknn7Wmh2aq93_WVq50dx9SOifIIBHvN8F6GVE,501 +jedi/third_party/django-stubs/django-stubs/db/models/base.pyi,sha256=77FrtJ8O86v8HFichcpzYvYxYetw0sS1ArGFdd10uNc,2400 +jedi/third_party/django-stubs/django-stubs/db/models/constraints.pyi,sha256=CXaVfiv0tWcDNPkxg8j7s1Be-teHkwMVuYZLZtMVasM,1089 +jedi/third_party/django-stubs/django-stubs/db/models/deletion.pyi,sha256=GEJcVeD0a_5KMj6u_kVxGQ1ezJj7rW5iccRZkQitN00,1149 +jedi/third_party/django-stubs/django-stubs/db/models/enums.pyi,sha256=Tu87vXSWCvRlSS_Sq3k8aptOXzmMF9JQj4Hk6pW3m3s,814 +jedi/third_party/django-stubs/django-stubs/db/models/expressions.pyi,sha256=QmaztG7k_UEThUZgtlDcwHxQASmil4xVM8ZfG6HQ-cg,8583 +jedi/third_party/django-stubs/django-stubs/db/models/fields/__init__.pyi,sha256=7cRcjZV7AVblUEXaAnQfzpnHX5yqCmenp8fyr9Ce_vA,13788 +jedi/third_party/django-stubs/django-stubs/db/models/fields/files.pyi,sha256=5EFmMvl68KKIC0O7BG7w4f-CEmLvtlxPHswHC8AtHTA,3640 +jedi/third_party/django-stubs/django-stubs/db/models/fields/mixins.pyi,sha256=myGDhO7COLmmouchsjRU57lQpXnJahzqaffXmya4r6c,453 +jedi/third_party/django-stubs/django-stubs/db/models/fields/proxy.pyi,sha256=HJvfUvqFTmnGwe3bZs0eHumpAvOdFYTJeX-KzOPvR80,161 +jedi/third_party/django-stubs/django-stubs/db/models/fields/related.pyi,sha256=6kgc4LA0vqc4JVg0Q9GRkHDuKr_6IfHwxVAoYFly2-I,9289 +jedi/third_party/django-stubs/django-stubs/db/models/fields/related_descriptors.pyi,sha256=foBVVlhgtaoOz6iRbAmh5Az18ZNbxND6gDncol863WA,3184 +jedi/third_party/django-stubs/django-stubs/db/models/fields/related_lookups.pyi,sha256=tPhBih5TNsVbpZ4_j-HM5Gt8tehukUcWxh-cnzARJw4,1500 +jedi/third_party/django-stubs/django-stubs/db/models/fields/reverse_related.pyi,sha256=EzSzqqZdtRcfUwzFDBZLZu8Nw1wbx5fyiyjzTfcSDEk,4045 +jedi/third_party/django-stubs/django-stubs/db/models/functions/__init__.pyi,sha256=zwujZ_mDvMGvDG7PZntnImLITlio1OMJlvuvH0hwmlk,2077 +jedi/third_party/django-stubs/django-stubs/db/models/functions/comparison.pyi,sha256=zMX1rKOTcfMhIyWmstp9DLfT06lDE7Yz-znCQVan-OE,312 +jedi/third_party/django-stubs/django-stubs/db/models/functions/datetime.pyi,sha256=YuYkZO0-rVO_6ChthbPlNHuB24w33RMSLAsipoUTn0s,972 +jedi/third_party/django-stubs/django-stubs/db/models/functions/math.pyi,sha256=4rGyuph9f9-2jlUnUDcCm2DKhTgkHvU4w2UOuGTzd_I,1222 +jedi/third_party/django-stubs/django-stubs/db/models/functions/mixins.pyi,sha256=HW4qO_JX_b1oScltR83xx0ekxCrXcvFMFZNCf6Hp5DY,100 +jedi/third_party/django-stubs/django-stubs/db/models/functions/text.pyi,sha256=8oIROhzzKZO1b8MvEmARBx4LgmfDNG-2ZRoNpYsK3yU,2228 +jedi/third_party/django-stubs/django-stubs/db/models/functions/window.pyi,sha256=poxMB0VKvi7XGx_8imATTi1aaOTGcEuaPLCv3zifFC0,702 +jedi/third_party/django-stubs/django-stubs/db/models/indexes.pyi,sha256=eKOgHZWF1_TUVZl3a8W3UpoDf3N73D07NU5Iw1qiaGE,1241 +jedi/third_party/django-stubs/django-stubs/db/models/lookups.pyi,sha256=BI1Ly6Sa_U1zDza00zTlC3pPF-bI4eKKev4L2bTlmqE,4475 +jedi/third_party/django-stubs/django-stubs/db/models/manager.pyi,sha256=w5myME9kNAigRReNYiUSNGjRzLUGWcwoqSfd49zmWH8,1796 +jedi/third_party/django-stubs/django-stubs/db/models/options.pyi,sha256=YC_LwZgqFGmUtfNhVmfRRTjSpTKpMCWbh8MxSXdLuoM,4993 +jedi/third_party/django-stubs/django-stubs/db/models/query.pyi,sha256=I04vyx6gHnK0Wm2atkwItk31rQBzNt2WwUx0u9YiB2M,9202 +jedi/third_party/django-stubs/django-stubs/db/models/query_utils.pyi,sha256=_FWehOpztavNcVxGBrAg7VFLdUNJeRigE7dqPt3-PDQ,3011 +jedi/third_party/django-stubs/django-stubs/db/models/signals.pyi,sha256=UvyKPck5LFYqOCeet0Z9HCxbtF9HKOofVak2mnyfPqM,850 +jedi/third_party/django-stubs/django-stubs/db/models/sql/__init__.pyi,sha256=mcrelxpwAAcq3e68wtJ3Y8vbb0JwRwln3GFy3PXTBf0,219 +jedi/third_party/django-stubs/django-stubs/db/models/sql/compiler.pyi,sha256=01_nE7RT8qgJbDsvul5mI-vifLFTQAwk1ETcJxc3U60,4712 +jedi/third_party/django-stubs/django-stubs/db/models/sql/constants.pyi,sha256=wqrVp2CmNwQGTE6G48hxopiddIaPjheG3aF0bNrfrxI,262 +jedi/third_party/django-stubs/django-stubs/db/models/sql/datastructures.pyi,sha256=aP3H2772NScQipXPMjWN8hReyS1cMtP35Tj0FOV0tTo,1909 +jedi/third_party/django-stubs/django-stubs/db/models/sql/query.pyi,sha256=HT_Qr-1R-rZvhpcYgMxLfWT3Xu82PmfvP7JyXNKMKgM,9072 +jedi/third_party/django-stubs/django-stubs/db/models/sql/subqueries.pyi,sha256=AvzqhSc6nWUiriu4jd62PAQt77fJ004mIDDuuzWpJu0,1828 +jedi/third_party/django-stubs/django-stubs/db/models/sql/where.pyi,sha256=SF_adSLaf14MqiKd-bjuWeX9eZFMBEXstMHnZS9hlT4,1918 +jedi/third_party/django-stubs/django-stubs/db/models/utils.pyi,sha256=ooqtUE_Tbh7fBMCgqzG9BkkycU8Pj3fD-tx3OEkqMuM,157 +jedi/third_party/django-stubs/django-stubs/db/transaction.pyi,sha256=8jqZMjzJCdLIVsBvkw_2elm7gagC_80GoW1SnhbmOs0,2032 +jedi/third_party/django-stubs/django-stubs/db/utils.pyi,sha256=EalrPIBxRawmO26UrJuJmUTeXCB8x24ehargXA4Gm9s,1244 +jedi/third_party/django-stubs/django-stubs/dispatch/__init__.pyi,sha256=SQUYnc7LrrXNZN_Ch5JdeGL9HDU-gaTK6G4Yxrr-rvc,78 +jedi/third_party/django-stubs/django-stubs/dispatch/dispatcher.pyi,sha256=mU0d9m4337GccA9rnNHlGm4xaj7ik5QY39cIeKK8VKM,994 +jedi/third_party/django-stubs/django-stubs/forms/__init__.pyi,sha256=4coCEFdYif8MNg0AXsb9poFP6Ih5OiB9Nytd_goGZmk,2957 +jedi/third_party/django-stubs/django-stubs/forms/boundfield.pyi,sha256=jiT0VOrFHPAwTTJYoFiHrtgZbVa8kxv8OjCwA_JuUSo,2308 +jedi/third_party/django-stubs/django-stubs/forms/fields.pyi,sha256=0F8m3ToEyszYNG0eB0VX-UgaYBlplgt2B1QPpmVAyc0,13037 +jedi/third_party/django-stubs/django-stubs/forms/forms.pyi,sha256=UHB2YbGoqojA9mu6mH7vFcHyFTgSa_y7cTcv7h2pF2s,3096 +jedi/third_party/django-stubs/django-stubs/forms/formsets.pyi,sha256=PWlKGK54Iu7khVNdt8nkqFnAHvlF1LkZCPttxU5jRpc,2397 +jedi/third_party/django-stubs/django-stubs/forms/models.pyi,sha256=B3ET3PjsUBJQWUasxTXCEOmoMsL9EOfdNTDnpchkzvw,10173 +jedi/third_party/django-stubs/django-stubs/forms/renderers.pyi,sha256=GSv3bIQiNtMy2NxLMWqZMSpGNQTJQUygIDYCecN41uE,724 +jedi/third_party/django-stubs/django-stubs/forms/utils.pyi,sha256=MgzO6rPlH29FFb7nkg-7ZSQYUjvQ92m-gqiS-bq5Z2w,1257 +jedi/third_party/django-stubs/django-stubs/forms/widgets.pyi,sha256=xO_oOqc02RaK07qV-g17bKBYlXyFhmRie6Nqrb5eq0o,6094 +jedi/third_party/django-stubs/django-stubs/http/__init__.pyi,sha256=tpXEfW48QUEnzOLVM9glXI6MLX3VX6_a-DmbzjIhvF0,988 +jedi/third_party/django-stubs/django-stubs/http/cookie.pyi,sha256=alynIOZMj8djeNCol-x2_mSUeNUtoO-stYZrboEDHnQ,102 +jedi/third_party/django-stubs/django-stubs/http/multipartparser.pyi,sha256=tq4nBvZN_mSeNC82urno1t2JhEKAqkXOPHpno5LIglQ,1986 +jedi/third_party/django-stubs/django-stubs/http/request.pyi,sha256=hxikJdIej6joG5W7mcj6h74Je5DvynLBpaoYT4vmxfg,3786 +jedi/third_party/django-stubs/django-stubs/http/response.pyi,sha256=JpWbTjhXgAGp7qotHfUTgHRLXoq49jL7YRDNrsjRi6Q,4994 +jedi/third_party/django-stubs/django-stubs/middleware/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/middleware/cache.pyi,sha256=2eNzy0MekvcFiR52WXQ1er12Gl3ZPNaYQBAesnRkaWM,1095 +jedi/third_party/django-stubs/django-stubs/middleware/clickjacking.pyi,sha256=KcQDinOFpxOnJt_bWJpFLk350ceGcODerjF3xcc4gEg,387 +jedi/third_party/django-stubs/django-stubs/middleware/common.pyi,sha256=KmN-G-bFLmSpqbZYXKF2rCWFbyT1thGKKQVscfCdLGQ,987 +jedi/third_party/django-stubs/django-stubs/middleware/csrf.pyi,sha256=-jr3w2lhfv_YPC74qNWZ_r8-CQ8jaJz9AU7c5te2FGA,1250 +jedi/third_party/django-stubs/django-stubs/middleware/gzip.pyi,sha256=If78A7dw_ILgnSCY2L8wkjNF2qqk7SkBVK17WAxVx7Q,339 +jedi/third_party/django-stubs/django-stubs/middleware/http.pyi,sha256=mdaMBPWaWuwjz0g400w1t5N3TkNGL9dG-TCmmkbAOd0,369 +jedi/third_party/django-stubs/django-stubs/middleware/locale.pyi,sha256=typ6K2fi1-vG64j0GVCSAXKCKzcpzmVE2bdDzoe8bes,423 +jedi/third_party/django-stubs/django-stubs/middleware/security.pyi,sha256=4ee7wQgRxiwpP_vwfG5IM68TYyCGpVPuTYEwp8uoM88,715 +jedi/third_party/django-stubs/django-stubs/shortcuts.pyi,sha256=WRzWWQatCnOL3yp2KLZ1H6b35J-eq3H6Ipk66YA2Y6s,1972 +jedi/third_party/django-stubs/django-stubs/template/__init__.pyi,sha256=87Omlyr7XLi5x8bbxTBby2htPXGn79eytzb2jguf2EU,648 +jedi/third_party/django-stubs/django-stubs/template/backends/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/template/backends/base.pyi,sha256=iHw-0xrm6rTptCjmp4N7VEZZRSHEGacxetcQotMVyBU,601 +jedi/third_party/django-stubs/django-stubs/template/backends/django.pyi,sha256=Ds2mLWa5DQirwlyvcy9SzMQVJDbiuvrV5jpYvTUbwbA,706 +jedi/third_party/django-stubs/django-stubs/template/backends/dummy.pyi,sha256=jhUAbJ7I1y6I56DI71u6geDjdndUJn1Fu4RkNIGh7AA,478 +jedi/third_party/django-stubs/django-stubs/template/backends/jinja2.pyi,sha256=ROpMhGxnaNlxWdYlvRWhvdEIFS7HQx1dB-x_FNK1WuQ,581 +jedi/third_party/django-stubs/django-stubs/template/backends/utils.pyi,sha256=Sbwk0bMuzGHJqvrhODl47g0H0n_LiFl7YYOwjHHWGVk,211 +jedi/third_party/django-stubs/django-stubs/template/base.pyi,sha256=eR590K7BCZtsab-Strdw8ju0Luft-YxFl7gme_g9pto,6030 +jedi/third_party/django-stubs/django-stubs/template/context.pyi,sha256=F1mgkncBKN_OeGuDO-dR3dZoYKVE2pxIoFu7ZnAcKUA,3208 +jedi/third_party/django-stubs/django-stubs/template/context_processors.pyi,sha256=bMVI13GgofaJnXyDHRgj09wVxMt0sA1DxawMrzwAtQc,640 +jedi/third_party/django-stubs/django-stubs/template/defaultfilters.pyi,sha256=fFNBc7tz39VsgxpHu5pOGyI1hyjQ44gwpSkObxGe6VA,3651 +jedi/third_party/django-stubs/django-stubs/template/defaulttags.pyi,sha256=hvxjkL-tRoFTg1JeScbEUeiMYF5HrnDw2XQpN4aY0QU,7237 +jedi/third_party/django-stubs/django-stubs/template/engine.pyi,sha256=fVxtt56BiOR-fLxGZtYFwvJD_ygYgu9Bhy5I39c7QbM,2158 +jedi/third_party/django-stubs/django-stubs/template/exceptions.pyi,sha256=XKMDInGTVL9Zr_exfpi6tS-zliwBuK5Dwg2hU9ZwAiE,596 +jedi/third_party/django-stubs/django-stubs/template/library.pyi,sha256=wqpD6-eFu6CHuraBEiO37sRG2Zhmkea1rv0I9eR2StA,3079 +jedi/third_party/django-stubs/django-stubs/template/loader.pyi,sha256=8O6_g_3h_zkRVqC5OdBbHG0MvIcPNWDr2qszobREPsM,620 +jedi/third_party/django-stubs/django-stubs/template/loader_tags.pyi,sha256=uhPtslJdwFOhqdsWtTzgOPgTFle7rf_eeQDVmmwFCuI,2363 +jedi/third_party/django-stubs/django-stubs/template/loaders/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/template/loaders/app_directories.pyi,sha256=tIrBH9Efep-8HYF_maLVfKPj8eGF_xY4ZxB22AmZsNI,88 +jedi/third_party/django-stubs/django-stubs/template/loaders/base.pyi,sha256=XNXihk_WvxirQJ20BpEPQZqcOms77836aOJYjCLwMl4,476 +jedi/third_party/django-stubs/django-stubs/template/loaders/cached.pyi,sha256=WROvtCtkV0dc4-6h2KISA3zO_h_73eo2id2bxRzLiz0,564 +jedi/third_party/django-stubs/django-stubs/template/loaders/filesystem.pyi,sha256=n3c4eQtiR0RBWkrPXGY8fi3bRTjEPlOnpEwawsQlojQ,433 +jedi/third_party/django-stubs/django-stubs/template/loaders/locmem.pyi,sha256=7P_Vsr1xLb_8tZ4QxEFF-31souSiyLHH0cKqlEcwIPA,354 +jedi/third_party/django-stubs/django-stubs/template/response.pyi,sha256=j1Fl0YSJTvostAVTfkMYy8g2KB9YlP3DBTaAZBMdLbU,2344 +jedi/third_party/django-stubs/django-stubs/template/smartif.pyi,sha256=PAKYMxW4Q6JY0l8TLmwR7B90xy8IYnMXTf7e6Atog_w,1267 +jedi/third_party/django-stubs/django-stubs/template/utils.pyi,sha256=keI7BX8onKPNwTqz9O1Osr1Rk7ZiyPIntQOvsPBo8Os,558 +jedi/third_party/django-stubs/django-stubs/templatetags/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/templatetags/cache.pyi,sha256=u-SCt-L2RlRBD9xG-pVTkwhThwJ7eNazTInZFAXHPWg,682 +jedi/third_party/django-stubs/django-stubs/templatetags/i18n.pyi,sha256=gca9DVbprTHJzIGX-GCcxEUViYDYfH10HffLu3xK9Ik,3176 +jedi/third_party/django-stubs/django-stubs/templatetags/l10n.pyi,sha256=8OV4XSr_rPNMhLOzxVu00SzC9VEnmt4OTRZmMZRqDZc,429 +jedi/third_party/django-stubs/django-stubs/templatetags/static.pyi,sha256=cUQ9iIHKXWgJXY2Qh9M1VXFgD_ASyNQbkVk4Unwg7oo,1170 +jedi/third_party/django-stubs/django-stubs/templatetags/tz.pyi,sha256=da1r04-aB0s1v4PR4d_TKredXpVkZR-kluQw1q07V-U,1166 +jedi/third_party/django-stubs/django-stubs/test/__init__.pyi,sha256=FxU0Xt4wJj1Jf3kdcL0lt-dpEDUN1wFe7LsyMiJvjjc,671 +jedi/third_party/django-stubs/django-stubs/test/client.pyi,sha256=A8dwFxbKc1rfjesp2FlOT__QXWPU0aEBOVZHmkyIFsA,5593 +jedi/third_party/django-stubs/django-stubs/test/html.pyi,sha256=k-d7X-HIXjPYhPpAggZQ9GR6MvkBGh1XmY5PnxGnuOY,1203 +jedi/third_party/django-stubs/django-stubs/test/runner.pyi,sha256=0fO4ry5uDw6o-wvnoBhSf8snCK6gd5yTQqw6b1RVJjc,5210 +jedi/third_party/django-stubs/django-stubs/test/selenium.pyi,sha256=CKW-XlD6SUu9pLyxUt6p8l5uYzN-RjkrGqnji-Dup7Y,368 +jedi/third_party/django-stubs/django-stubs/test/signals.pyi,sha256=yBlj9hfd9baDs5gt0x6Ymdg9L4cOtqPe1TRg63VXxKk,986 +jedi/third_party/django-stubs/django-stubs/test/testcases.pyi,sha256=rUMMlG0haG8l9frb1RRgLMZZtRLKEV5a0JAsLJa7jOU,8273 +jedi/third_party/django-stubs/django-stubs/test/utils.pyi,sha256=GmKL_4fBsAa__tn0gXHPJeIq1MsiUN9utqm9t2fsJ-E,5436 +jedi/third_party/django-stubs/django-stubs/urls/__init__.pyi,sha256=nQLDuDGK7FnjTv35h5oWaHGxEj7LUxTpXHmM1bPV1n0,1197 +jedi/third_party/django-stubs/django-stubs/urls/base.pyi,sha256=RtQApc1gUSPpnAw1gm3HqvNai7lYb0k3r2kS6wcKMk0,887 +jedi/third_party/django-stubs/django-stubs/urls/conf.pyi,sha256=7wH1fulW8v9yZlqEy4yQFSGfVznjPJM1DC5V-ZiSoEY,224 +jedi/third_party/django-stubs/django-stubs/urls/converters.pyi,sha256=--viS0cndpAqzNeusl7RLKBVhsrPIoOikVgmQDNxGxc,827 +jedi/third_party/django-stubs/django-stubs/urls/exceptions.pyi,sha256=4UVr_L5SexIY1ok1zhdDAq1v1iyt7l0NiDdOu7i07lg,102 +jedi/third_party/django-stubs/django-stubs/urls/resolvers.pyi,sha256=gvKPNzAwWo1kE-wvLFhWk83C7qm1hsuF7CPPzCv9ypQ,4029 +jedi/third_party/django-stubs/django-stubs/urls/utils.pyi,sha256=aHMnJJ_yLIRc1jFBsuwInS1i07sg_TNCRFfpBdVliWA,168 +jedi/third_party/django-stubs/django-stubs/utils/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/utils/_os.pyi,sha256=o_1Okw2HuOcaOrD-_SzHIgSVDNmBShknGwqd5uat8ps,307 +jedi/third_party/django-stubs/django-stubs/utils/archive.pyi,sha256=uYtbaGft9lKuO5eEfMBWnmOX3lD-ztopvgNhLZaP4QM,1043 +jedi/third_party/django-stubs/django-stubs/utils/autoreload.pyi,sha256=C8KHbfg9nGyhfbNa56RXXP0v1rlIkTUvYw1zkmv3bOk,2702 +jedi/third_party/django-stubs/django-stubs/utils/baseconv.pyi,sha256=Myy9JlfWTJ29J8Oyiq-upqag-VfsPHg3jsuq7HHU9iE,593 +jedi/third_party/django-stubs/django-stubs/utils/cache.pyi,sha256=9ss0oBvIs1-gBExjVldryr4hESowSSeQdShVgMZdugs,1338 +jedi/third_party/django-stubs/django-stubs/utils/crypto.pyi,sha256=uccSZ_WsmWkCQ_4V6SQXOJd8NUn3FdyjnUbcP5K1I0U,543 +jedi/third_party/django-stubs/django-stubs/utils/datastructures.pyi,sha256=CUquMJUCgGCjeVUk3QSx9vVcZjmHKJsgVI3Vjqti1Zk,2624 +jedi/third_party/django-stubs/django-stubs/utils/dateformat.pyi,sha256=KLHBQh9aWgLYWbeqw2kiyQauGNDiKmadeMmltidGlrk,1782 +jedi/third_party/django-stubs/django-stubs/utils/dateparse.pyi,sha256=PX8c8e3ma-IOaj-F0RrBF4XTTVngSfkGNKt24AMy4qo,425 +jedi/third_party/django-stubs/django-stubs/utils/dates.pyi,sha256=eDDUi7Z7G3VU8h_6q4Z63l_7L2zgcyYuTE3yf-GSt-I,181 +jedi/third_party/django-stubs/django-stubs/utils/datetime_safe.pyi,sha256=fXQJdx9zqee-Jac6dhydWUMKvz8JzQYnKhwPsxiDnfY,341 +jedi/third_party/django-stubs/django-stubs/utils/deconstruct.pyi,sha256=6IL3eW3s2avXEeGE_oJW3Mxg6Hk3387euuFekBeHY0A,105 +jedi/third_party/django-stubs/django-stubs/utils/decorators.pyi,sha256=AEPbjOpQtJGpOjwrri-6wV2gzUIvNZ5EIVJVsTEushk,935 +jedi/third_party/django-stubs/django-stubs/utils/deprecation.pyi,sha256=yGFYy_fyf37-YpyWdalYAdQXCX8YLVCOgu8lQVVBxpk,1345 +jedi/third_party/django-stubs/django-stubs/utils/duration.pyi,sha256=W8vapfROyCEor7wRgPdoqY7cpLg8kFEqiAKHqjEYfUk,198 +jedi/third_party/django-stubs/django-stubs/utils/encoding.pyi,sha256=e-_D5ZVUcDKL6yXDbcj1BpXsovmfiOogb9ioT1j0vl0,2416 +jedi/third_party/django-stubs/django-stubs/utils/feedgenerator.pyi,sha256=fxf5z74HoHuKF_W1XVOxHMs1VbbD233GHN1o2pkMl68,2677 +jedi/third_party/django-stubs/django-stubs/utils/formats.pyi,sha256=WLkueQDaB2Gg2MTb_SxnFkpQpR49O6ogaZEKhy2JGFQ,1271 +jedi/third_party/django-stubs/django-stubs/utils/functional.pyi,sha256=_k802K-xEtUcEFBrH09Sxsh_7ObZAARVF12UAMGqkug,1923 +jedi/third_party/django-stubs/django-stubs/utils/hashable.pyi,sha256=l25xFctJPg50l6Y129bvdAFJc8QqefkXx40amy6LkhU,66 +jedi/third_party/django-stubs/django-stubs/utils/html.pyi,sha256=uRHTbSOfG3lG2pwS5CdKwRmnfAC2DSgG7UoWkrXZRFM,1366 +jedi/third_party/django-stubs/django-stubs/utils/http.pyi,sha256=zUCz7anMbyPWISp458GPg3TfnE6iehGn1LPBsPE6ru0,1544 +jedi/third_party/django-stubs/django-stubs/utils/inspect.pyi,sha256=4ifnyH12SSqy2OzqUkuJxvFGTkiirblPt4eo1Xu8GyI,391 +jedi/third_party/django-stubs/django-stubs/utils/ipv6.pyi,sha256=_VLusX7GiWNffbeDTqd_jnJPOdxyVS179DFuB61CjhA,168 +jedi/third_party/django-stubs/django-stubs/utils/itercompat.pyi,sha256=UpxNkxuzzgXwrBfRqew_8T5D5tx5WBsCTP933uGVHwI,61 +jedi/third_party/django-stubs/django-stubs/utils/jslex.pyi,sha256=_R3st8851lEA_sUS5i_Kneche-ZPrCtpIsBtzFQZS_U,748 +jedi/third_party/django-stubs/django-stubs/utils/log.pyi,sha256=yVMX_aEDnocTx6PpJgpvOLVqD-cCmBz9-DnFDpl5ckI,1501 +jedi/third_party/django-stubs/django-stubs/utils/lorem_ipsum.pyi,sha256=hM07Nj_QipTFShSP7PrhQxlwAlYcgcEpH3vXRF_gzcA,248 +jedi/third_party/django-stubs/django-stubs/utils/module_loading.pyi,sha256=sd6wFSrhzvHm648lgnvfdcmkR3PC8SPC925rP83DNkw,247 +jedi/third_party/django-stubs/django-stubs/utils/numberformat.pyi,sha256=tGq_aAmrf_lkWdZ4XUMlf4h0DUtc185bwfgJefaXYkw,344 +jedi/third_party/django-stubs/django-stubs/utils/regex_helper.pyi,sha256=REFgvf8HYnE-XsiepvPLjd2HZyMWlUJDC2ILtUDS0Ws,658 +jedi/third_party/django-stubs/django-stubs/utils/safestring.pyi,sha256=G4jCaiDDv7gb75b9pJFVMOVRZfnECaDk0JYn8jZmxag,629 +jedi/third_party/django-stubs/django-stubs/utils/six.pyi,sha256=fGwtJGbkPulV7pzqEeAdgzVv6z2mDiFtgtios6tWzS4,3408 +jedi/third_party/django-stubs/django-stubs/utils/termcolors.pyi,sha256=gqCN0etzSjG8bN21K_j1d1Jw0UaluPrlEznqx36bGek,517 +jedi/third_party/django-stubs/django-stubs/utils/text.pyi,sha256=ZbmkpySA5CNdPKorDNm5WHh0n0hcF7pp8vI_35z2Bw4,1583 +jedi/third_party/django-stubs/django-stubs/utils/timesince.pyi,sha256=7qwEhIWEQeKk71J8pQJzr7QCljggRtDHEs8-GUiiLOU,362 +jedi/third_party/django-stubs/django-stubs/utils/timezone.pyi,sha256=S6fV8Ub-gZuQzIPWx7RSlugjjcWQFDJVlPxFnq280tE,2714 +jedi/third_party/django-stubs/django-stubs/utils/topological_sort.pyi,sha256=vaxSleSm6rJ0axW_bodRRXA6a82056AxplxL7hbdwTY,297 +jedi/third_party/django-stubs/django-stubs/utils/translation/__init__.pyi,sha256=Xb8uG790oIzlgTor_Czpto7O1o3slls-Yb1gRKqzFvc,2232 +jedi/third_party/django-stubs/django-stubs/utils/translation/reloader.pyi,sha256=WysOUTcv4mLco3UizcmXdD5Im-HrD8nrBy71A_3Phd8,299 +jedi/third_party/django-stubs/django-stubs/utils/translation/template.pyi,sha256=H4yDCnEfEMFUE4OIMg1FJYn9cvj5CbQRv1zeokTe6QY,235 +jedi/third_party/django-stubs/django-stubs/utils/translation/trans_null.pyi,sha256=x7RjAzIctThGOsg6ZKE0NpO6ekKpl1TgsCxaop9G864,729 +jedi/third_party/django-stubs/django-stubs/utils/translation/trans_real.pyi,sha256=uXK53giQyz6hE5WsJPBCkBepZjw9Ljxhwhqk7MLYo_I,1815 +jedi/third_party/django-stubs/django-stubs/utils/tree.pyi,sha256=sR1J0tJzZIGobNu8-XiJ2qr94SlwiM164sF8lD5-8aE,799 +jedi/third_party/django-stubs/django-stubs/utils/version.pyi,sha256=kwDgHzvG-PW5Gs-2ch8lTduLItyo4asFa509YOQd15A,516 +jedi/third_party/django-stubs/django-stubs/utils/xmlutils.pyi,sha256=wGOSksvFX7wd6dl7Yoz836aEyZ-D6jxi4R9kBaaeEtY,433 +jedi/third_party/django-stubs/django-stubs/views/__init__.pyi,sha256=aq7HMnOVrisbvfSuGERsghE8UbQzyGkJq-QpU0Tcv9w,39 +jedi/third_party/django-stubs/django-stubs/views/csrf.pyi,sha256=IWUdDCiJjyVfGZHpN0fHH6nu7yYmVbXyDvWg-Bea23Y,274 +jedi/third_party/django-stubs/django-stubs/views/debug.pyi,sha256=4oGmtVDkDja_yZdqDmfpXGR7-UncwY28nsYv9posmrY,2781 +jedi/third_party/django-stubs/django-stubs/views/decorators/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/django-stubs/django-stubs/views/decorators/cache.pyi,sha256=HNIBcnkLIFOVKkhlD6NCF8hX3MWGBr6pclv27ty-1LY,303 +jedi/third_party/django-stubs/django-stubs/views/decorators/clickjacking.pyi,sha256=Ok9pqjwji2pR7WQPLtZ1CrRu0W9ahdyb-JaP6c4wq5c,247 +jedi/third_party/django-stubs/django-stubs/views/decorators/csrf.pyi,sha256=AWLjm6s5aCI5QnU96JbH6Y5UZxMrmpY227yLDq0jf78,477 +jedi/third_party/django-stubs/django-stubs/views/decorators/debug.pyi,sha256=wZ1YJ1dKztkQ6wd3ED0G6G4l4WuHSRcABy5qOlmFYCo,157 +jedi/third_party/django-stubs/django-stubs/views/decorators/gzip.pyi,sha256=Xjc5S0snvNoApW9JpvSQUhGypsjoCUE8bC0HwaVdL4k,129 +jedi/third_party/django-stubs/django-stubs/views/decorators/http.pyi,sha256=ZTJ_lLWuXIdnX80y1OA230O8ep50pxlSa2acazv-1gw,576 +jedi/third_party/django-stubs/django-stubs/views/decorators/vary.pyi,sha256=yKYeW6Fa4M_bbOYU0evNeMS_0ly-9ZCJonlkD_VqhRQ,181 +jedi/third_party/django-stubs/django-stubs/views/defaults.pyi,sha256=qnKCAJ0gMNl7y0DbyQRDVDm0WnR7iWhIAFZgsILVQqY,814 +jedi/third_party/django-stubs/django-stubs/views/generic/__init__.pyi,sha256=RMzs5h07bv50ftLnRSYnrWOJXkth7PTyU5odn0GCw68,637 +jedi/third_party/django-stubs/django-stubs/views/generic/base.pyi,sha256=EcfZBLJd_E_yktw5j8YYeCKO4OKdOnY8Pns7x9XT2BI,2239 +jedi/third_party/django-stubs/django-stubs/views/generic/dates.pyi,sha256=sV3GQ35fJCDh5I19zVFQkvEHhkIUeVyHqQ9UBRRBVRM,3771 +jedi/third_party/django-stubs/django-stubs/views/generic/detail.pyi,sha256=PNnYL1e7LJxYr71wI93v0kuwfra8YGOHZvD6-1wIodA,1090 +jedi/third_party/django-stubs/django-stubs/views/generic/edit.pyi,sha256=-ngfSEOCkreuUeU0XcL9s1eNWnZHlPw6VlxdUYvix5E,2234 +jedi/third_party/django-stubs/django-stubs/views/generic/list.pyi,sha256=-Hj-OCJw6BgmXFsUgNHSJfeeECrpyqbBWShSLxJaveU,1625 +jedi/third_party/django-stubs/django-stubs/views/i18n.pyi,sha256=uL-yCYqHyEFkfL2He_UTt_OnBnom8TlqhAquvxMd1Sc,1257 +jedi/third_party/django-stubs/django-stubs/views/static.pyi,sha256=tX0ADulNf2Yh6fJhKh6PSS9a8AsAt1gijAMu2iehhP8,461 +jedi/third_party/typeshed/LICENSE,sha256=tBt49WKi5lt1yrRDVDNfb0NdTvcwZVCWAKyRDPTiL-A,12658 +jedi/third_party/typeshed/stdlib/2/BaseHTTPServer.pyi,sha256=0VJgE581N-I4xIycM3XdZA3Ag9e5du0W2tzWwWvuPjY,1747 +jedi/third_party/typeshed/stdlib/2/CGIHTTPServer.pyi,sha256=pv5Xb3udKa7NXOLe3MI__b_zP6a0wmYuFmOYP_N3kuc,187 +jedi/third_party/typeshed/stdlib/2/ConfigParser.pyi,sha256=lFmtKNTfILcHhIRgVu3-d0jKhaBtggGm9qSjqmIBH-Q,3869 +jedi/third_party/typeshed/stdlib/2/Cookie.pyi,sha256=Ke_iU9vRRHFMkJrben9gxXLss3nTR0bw5ZPvN_abYdc,1342 +jedi/third_party/typeshed/stdlib/2/HTMLParser.pyi,sha256=4mV-tgsweFHjVH6VFVR6pwxsUJt8Ttszq4Ka3gtSOPA,1064 +jedi/third_party/typeshed/stdlib/2/Queue.pyi,sha256=jVv-Ahqmn2SC748u_-61UeRcrK4cQXiQbm5lVgxCDeM,895 +jedi/third_party/typeshed/stdlib/2/SimpleHTTPServer.pyi,sha256=kgfdaGTvBrw515C2tKgPM6PBG1vx8Uym30edR9hVnoU,648 +jedi/third_party/typeshed/stdlib/2/SocketServer.pyi,sha256=4V7NHwSn6jAPavRtz1w8bBwPNNXvGL02UYQAlm6bA-c,4559 +jedi/third_party/typeshed/stdlib/2/StringIO.pyi,sha256=GBqd-c4z25y99oSUVcsEMh98LoMruu1ZOb0AE5UOtzk,1146 +jedi/third_party/typeshed/stdlib/2/UserDict.pyi,sha256=h0miLRBjzRA6LL-sfVhN5lIN19AjToRaByhueNrVxoQ,1663 +jedi/third_party/typeshed/stdlib/2/UserList.pyi,sha256=aDxMr5yhCHPZT-SacAYmju5xaCquHi1Nk_z_rZRvHN8,630 +jedi/third_party/typeshed/stdlib/2/UserString.pyi,sha256=3391hc_oGw3FFPX1zlPbil5GVZbqKqlLbGup-08BnLE,3844 +jedi/third_party/typeshed/stdlib/2/__builtin__.pyi,sha256=-CqbxZmdacwEbwoRuybLasluru9S8wSSaWEfq57ZfU8,48853 +jedi/third_party/typeshed/stdlib/2/_ast.pyi,sha256=36MwM7zIYwSWhw4G2sLtd-L8I9hAZs3lUEw4n_J88a4,5726 +jedi/third_party/typeshed/stdlib/2/_collections.pyi,sha256=cMqBXKAr1o6kdqDM8RkZNclFduh0s-QAv-jUmpUYHc0,1430 +jedi/third_party/typeshed/stdlib/2/_functools.pyi,sha256=rDxIud_hO61m7kUqrvpkJT4mwoiyOvTxqMejOjY5nhM,576 +jedi/third_party/typeshed/stdlib/2/_hotshot.pyi,sha256=ffVl3P1C-rfN5DN6smwzDGDOoUNEasmNkrMKo5mnTbc,635 +jedi/third_party/typeshed/stdlib/2/_io.pyi,sha256=JoQeS8uF6Nz0yz_qbFHinI5YarcMPl7fkWpWv5aHwtQ,7016 +jedi/third_party/typeshed/stdlib/2/_json.pyi,sha256=VGD_NDPa30SGzHZ17hKmgnCk91AhGeOP_klprIkApio,226 +jedi/third_party/typeshed/stdlib/2/_md5.pyi,sha256=pGqwb01a_RcSP1QRE8XtVPB0RKaoJOGuRdVk6pwvEag,300 +jedi/third_party/typeshed/stdlib/2/_sha.pyi,sha256=32F3_E2nGplztFti0fx5GwfPqobLiyg2rtTLHopfCw4,348 +jedi/third_party/typeshed/stdlib/2/_sha256.pyi,sha256=1Z5g4wLOL9-z6gasal2kMoBb7yBHPJMFSCgPts_GTRM,632 +jedi/third_party/typeshed/stdlib/2/_sha512.pyi,sha256=6AyOELlW_oDueP9i8yvirht0BdJO0itNx9-deuMYCeA,632 +jedi/third_party/typeshed/stdlib/2/_socket.pyi,sha256=wVuvH1tMPqjPe89bG7pmVsuaXZJfSdvw4qWRD9geQBU,6286 +jedi/third_party/typeshed/stdlib/2/_sre.pyi,sha256=mcck_gioJ8R-V4cW7y7GuCuOihgfKTGcP8RSV_eO6l8,1934 +jedi/third_party/typeshed/stdlib/2/_struct.pyi,sha256=cAuIIq62g6PJsL_G5z16j4v5aTGqYDM6j69kyRcPIsM,767 +jedi/third_party/typeshed/stdlib/2/_symtable.pyi,sha256=LLeUYtJkj_EBmkpwSffnopwY_lsPpopzvM-kXQOwg_I,677 +jedi/third_party/typeshed/stdlib/2/_threading_local.pyi,sha256=1y8WTQOMyBV6cA0BZRbm0Q7bDtCDciIspnuKwYHTrd8,319 +jedi/third_party/typeshed/stdlib/2/_winreg.pyi,sha256=4Q60Z4nLxuUQi1LRbBAKKdAOy0Ir5b-2Bk-RDhYYsU8,3696 +jedi/third_party/typeshed/stdlib/2/abc.pyi,sha256=tzQd1K85KZQBh7b4lulFwkRc2GitR7uplj5toNIJsCA,1147 +jedi/third_party/typeshed/stdlib/2/ast.pyi,sha256=fkVIuq6FiMNU3VMVnZmttTt4-OLZ-CfNMoGexWCZLh0,1199 +jedi/third_party/typeshed/stdlib/2/atexit.pyi,sha256=Zx4nX1WyFC_XDP2Pp17lXrMMJiaCjCut0VWHgtXWBok,117 +jedi/third_party/typeshed/stdlib/2/builtins.pyi,sha256=-CqbxZmdacwEbwoRuybLasluru9S8wSSaWEfq57ZfU8,48853 +jedi/third_party/typeshed/stdlib/2/cPickle.pyi,sha256=FR7Mq3gugr2YQWV0TgHkrE0e-UgEB7OrklOlOg66vdc,795 +jedi/third_party/typeshed/stdlib/2/cStringIO.pyi,sha256=PQQ1dSSTlVi-NTl8Qv0IKwYATynJuLz14g6b27C0G-s,1870 +jedi/third_party/typeshed/stdlib/2/collections.pyi,sha256=5psNp9WgNtInnDK2-chmkny_9NkQcbG7TeFsQZCiX6c,4913 +jedi/third_party/typeshed/stdlib/2/commands.pyi,sha256=nGsS9_5mqYgZqIx-g-A0eQ4Vd9WQWvp53Fv8PEpAeic,329 +jedi/third_party/typeshed/stdlib/2/compileall.pyi,sha256=gx2rOct4wjjTrX8pbrd0ZR3CPNiQkjP1OQ1LARDIftw,628 +jedi/third_party/typeshed/stdlib/2/cookielib.pyi,sha256=f3yMHlsN0iK7oqGhGUD2_0lEHw3mljQ00itXN6T6Qwg,4716 +jedi/third_party/typeshed/stdlib/2/copy_reg.pyi,sha256=-GmSIudewg4xblZXLSvErhpunAMi1bsqdQUEXujpFlI,739 +jedi/third_party/typeshed/stdlib/2/dircache.pyi,sha256=cOqJhmlvq7Vl92FUWcfQGX7aedsgjCOYPTuyZEKD2uo,273 +jedi/third_party/typeshed/stdlib/2/distutils/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/archive_util.pyi,sha256=WaUu32XUCstlA4zE0WS-PvA9UF0mdyuDgDbZZd1LN0A,447 +jedi/third_party/typeshed/stdlib/2/distutils/bcppcompiler.pyi,sha256=fge2cMbG4jp--o0I2zNcwykh24tJWZtk6leQgAH2NJw,78 +jedi/third_party/typeshed/stdlib/2/distutils/ccompiler.pyi,sha256=N38wYG41RyiEuowx_ZvpZIqVVZYiz1NGcGHHDJ9MbWs,6449 +jedi/third_party/typeshed/stdlib/2/distutils/cmd.pyi,sha256=SywVh_oj7uAo1tmMjytIafcLCJMfAtMFFZe9Q3vA6Yg,2817 +jedi/third_party/typeshed/stdlib/2/distutils/command/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/bdist.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/bdist_dumb.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/bdist_msi.pyi,sha256=sDSqH7TRcOiXC5S4VXxJ_YHB-WFPpa1fo8F8g5XeV3Y,182 +jedi/third_party/typeshed/stdlib/2/distutils/command/bdist_packager.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/bdist_rpm.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/bdist_wininst.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/build.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/build_clib.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/build_ext.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/build_py.pyi,sha256=z4m9RU6PID-qalM7jzvc2mIWMwk0saczeEmqq9qleH0,181 +jedi/third_party/typeshed/stdlib/2/distutils/command/build_scripts.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/check.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/clean.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/config.pyi,sha256=DV1oMIztVDdk46W43HGioP_n6b3x9FgSqvFr2rwPVoY,3059 +jedi/third_party/typeshed/stdlib/2/distutils/command/install.pyi,sha256=BCv1Lbe-6AHxtaHuhIGd3nOtN-efRmZI-xvpxwKd4Fk,338 +jedi/third_party/typeshed/stdlib/2/distutils/command/install_data.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/install_egg_info.pyi,sha256=WcnLycNSSWSZ8Z_vHIohu0-3qKnCih2qoJaGBPtjQGY,380 +jedi/third_party/typeshed/stdlib/2/distutils/command/install_headers.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/install_lib.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/install_scripts.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/register.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/sdist.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/distutils/command/upload.pyi,sha256=Sv7tBpCnOzInTz9nVhZeo6Waz4brX1PtS51GKzQNwOU,296 +jedi/third_party/typeshed/stdlib/2/distutils/config.pyi,sha256=M4b9_7PKsO74DhsIBFoTAB5I99hXrlBJcjxghWhI8vc,523 +jedi/third_party/typeshed/stdlib/2/distutils/core.pyi,sha256=lq1S8lRqeGqK6U3z7_UsVg2TR7xbx7Z2YXJ0ekIo9Vk,1688 +jedi/third_party/typeshed/stdlib/2/distutils/cygwinccompiler.pyi,sha256=Y7qhVOqXrkPT0yyQnudNCtTzBYC2lzS38HB5Mh45zEI,138 +jedi/third_party/typeshed/stdlib/2/distutils/debug.pyi,sha256=7_zuriUBqHbc62x7tCONq7LQXLuK_hCaBK0laoR3HeY,12 +jedi/third_party/typeshed/stdlib/2/distutils/dep_util.pyi,sha256=QCheHEDF7waISF42_aaumqvVOIcTw-yh5e5-CbPvQ2o,252 +jedi/third_party/typeshed/stdlib/2/distutils/dir_util.pyi,sha256=0nHuLCqZ36gvDVaF6PoC76_JOC3v6P_310eFauW1ZDM,555 +jedi/third_party/typeshed/stdlib/2/distutils/dist.pyi,sha256=DtSDKCXM6UsXnoN7zpt_vxjOgLJu8qeFY2RBcwaB8RM,508 +jedi/third_party/typeshed/stdlib/2/distutils/emxccompiler.pyi,sha256=FxHjF75rMcBNR4odfAyfbbayTXE-2tfbAkHtKxGgYHw,90 +jedi/third_party/typeshed/stdlib/2/distutils/errors.pyi,sha256=l1W_FgoP9L-D-hEPFA2BzZuybjN0lV4WBXl0VJ-k7J8,852 +jedi/third_party/typeshed/stdlib/2/distutils/extension.pyi,sha256=tsIOARiVdGDcvsnxVFxBa80iydfox_Bt6EGqlJsL7kU,706 +jedi/third_party/typeshed/stdlib/2/distutils/fancy_getopt.pyi,sha256=ai-0E83HnTOcYCWO9zq4tDVvnef04SRRoa-F8baso_o,859 +jedi/third_party/typeshed/stdlib/2/distutils/file_util.pyi,sha256=RFpiizXMdhhcji98pIscfHW4r6i9KLzM2D15gA6_eow,439 +jedi/third_party/typeshed/stdlib/2/distutils/filelist.pyi,sha256=-WeYFFKsEUUjPvbzeZbVCOKkPV-oqc3RoZvN2SB1VOE,20 +jedi/third_party/typeshed/stdlib/2/distutils/log.pyi,sha256=9vvQVRer-_-S5lcV7OHF1Ptr1N3npjKvzTVXReSpZKA,863 +jedi/third_party/typeshed/stdlib/2/distutils/msvccompiler.pyi,sha256=qQLr26msfhjz-omJutWcRHik3shLh1CIt7CDI3jBd3I,78 +jedi/third_party/typeshed/stdlib/2/distutils/spawn.pyi,sha256=iDdi2WvST9yeFDlKWy0Wlye-x67W-ah5nta7EuRW2W4,227 +jedi/third_party/typeshed/stdlib/2/distutils/sysconfig.pyi,sha256=FSdBoSTsVvKAF5D2lkWBwxH15ockfFZv6L06mrgeAb0,620 +jedi/third_party/typeshed/stdlib/2/distutils/text_file.pyi,sha256=vCQLwggDaocAqqR7v1WJjOeS_wgxqjI5xDkyxHJlzcw,716 +jedi/third_party/typeshed/stdlib/2/distutils/unixccompiler.pyi,sha256=R3VKldSfFPIPPIhygeq0KEphtTp0gxUzLoOHd0QoWW8,79 +jedi/third_party/typeshed/stdlib/2/distutils/util.pyi,sha256=KSpX8rQ6qJXqToJBKdwhlpe-jd1QQcacg7wsH_6dKXo,829 +jedi/third_party/typeshed/stdlib/2/distutils/version.pyi,sha256=PU7GKbMl1ivgTKVK54jB2fgIufr_hDCUgzjYyNXt-4E,1160 +jedi/third_party/typeshed/stdlib/2/dummy_thread.pyi,sha256=755Cy6AXyEo3RowYk0pQm5I5mkAIE3yQrkWImnrlHOA,794 +jedi/third_party/typeshed/stdlib/2/email/MIMEText.pyi,sha256=4Hjv1f-LZwoj-ihndmbQNHdwpjOy6wUOJoKS_axJmNo,159 +jedi/third_party/typeshed/stdlib/2/email/__init__.pyi,sha256=iUDv6ttU1qT359eOAubG1JtxNmrJGu8QxH_aXPvOz9w,270 +jedi/third_party/typeshed/stdlib/2/email/_parseaddr.pyi,sha256=oqGaUf13WZALSq7cyULZ0c_6iFKjH8rdnAfAkm6y3Hw,1072 +jedi/third_party/typeshed/stdlib/2/email/base64mime.pyi,sha256=Qb1Q4NHIbSJOcsZ8vUBqaPT-s6lWpj-YD1kI9DI6Xfo,303 +jedi/third_party/typeshed/stdlib/2/email/charset.pyi,sha256=VVEUOTe1XZ824-FhBuIBrSCB16hMAnQ1Ygseu3Noc_Q,902 +jedi/third_party/typeshed/stdlib/2/email/encoders.pyi,sha256=s8kQE5AG1wvh0h0qbNn3_As6ExYQccVdg6Bx2PKGu8E,143 +jedi/third_party/typeshed/stdlib/2/email/feedparser.pyi,sha256=cKLfhKboxZeJxceH5e_broSJZDa4teMu_ZJvZRhREQU,536 +jedi/third_party/typeshed/stdlib/2/email/generator.pyi,sha256=TOAFU4Cb0_a3EitMT62JWGtcoGuvgrfKlbWpNAmwEuA,377 +jedi/third_party/typeshed/stdlib/2/email/header.pyi,sha256=sCk_MfWl5P_bc5v9302SubX0hqgODtlpJsnPb6h-eC8,457 +jedi/third_party/typeshed/stdlib/2/email/iterators.pyi,sha256=vPq5eJF8HBwFQ1hS--niEmurSl4x42YOrU65TxKk0Jc,256 +jedi/third_party/typeshed/stdlib/2/email/message.pyi,sha256=M3XzQbdji1k8_hygt88priwEMJqWKRixQsN4qDLmfeU,1950 +jedi/third_party/typeshed/stdlib/2/email/mime/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2/email/mime/application.pyi,sha256=zlzwumM16ipPeDnDgS11A0hzIJ1LEegxL7g0-I3dSWw,371 +jedi/third_party/typeshed/stdlib/2/email/mime/audio.pyi,sha256=GYtzGiC2dTHTPD3Cm3uIUlBAQ_25NK2BsbbcuDbLZiU,176 +jedi/third_party/typeshed/stdlib/2/email/mime/base.pyi,sha256=lG1Re_xRHsaw4WRUnLh1Jyneb4M6m8kxqa0NUfwuONg,128 +jedi/third_party/typeshed/stdlib/2/email/mime/image.pyi,sha256=-HVa8k6js_9-sGt5jFg2SF-UjZ6cLP53T9GvRIVI63s,176 +jedi/third_party/typeshed/stdlib/2/email/mime/message.pyi,sha256=OwJjEUiejk2bc9FGqgbvz8Q6ZgQyAg9gphhDXPyXsLU,147 +jedi/third_party/typeshed/stdlib/2/email/mime/multipart.pyi,sha256=1pTSK5lU6L5AJG5H35PTIQtHYIplMoipa7Kkd_m9HNQ,159 +jedi/third_party/typeshed/stdlib/2/email/mime/nonmultipart.pyi,sha256=C9WcyywCzQqkL9MPpSlWHgChmP04r0rrWVw3VlSVHQo,107 +jedi/third_party/typeshed/stdlib/2/email/mime/text.pyi,sha256=4Hjv1f-LZwoj-ihndmbQNHdwpjOy6wUOJoKS_axJmNo,159 +jedi/third_party/typeshed/stdlib/2/email/parser.pyi,sha256=9QChl7gsm0KPwZHUYy5tR_kZkmQpdSnxCwuZTnp9ceo,415 +jedi/third_party/typeshed/stdlib/2/email/quoprimime.pyi,sha256=ZRJzHi-3Fszfa8nRpz6EpGYZdSpLyGc4K3pxr1uyMUA,490 +jedi/third_party/typeshed/stdlib/2/email/utils.pyi,sha256=Znm4b4ImavrJrwGc_6PcBqyhqo3zIbqDc7gSS_6NU0w,760 +jedi/third_party/typeshed/stdlib/2/encodings/__init__.pyi,sha256=q95Eqw-U2YICiPawMyarXy1iWCXsz1wV_793ZSg_4ek,184 +jedi/third_party/typeshed/stdlib/2/encodings/utf_8.pyi,sha256=tgCdNX8etJQWYWmOYAIZhK8lcYm_Kn67kylKJp0SgUo,573 +jedi/third_party/typeshed/stdlib/2/exceptions.pyi,sha256=i3AvRM6Osg9r_q5outQ4hn7COs0fyhsJizSH5M72G7k,1756 +jedi/third_party/typeshed/stdlib/2/fcntl.pyi,sha256=4A81Md4YTEt71UFrJWD8E4Pp7iQGMiXN5geQ1-AEuA4,1580 +jedi/third_party/typeshed/stdlib/2/fnmatch.pyi,sha256=8kgI-ZZR0lhAGSuQk0M0kt3cYrYRx29bwhIg9ESvLbs,348 +jedi/third_party/typeshed/stdlib/2/functools.pyi,sha256=t_xmkNhLllvRSRG-5X1uP9sW-IHgkAns8ts0uTLllrw,1180 +jedi/third_party/typeshed/stdlib/2/future_builtins.pyi,sha256=vkVYaUei63-XJqSnDDLn7KwkUyLpVwbP01ZGTRWlySc,194 +jedi/third_party/typeshed/stdlib/2/gc.pyi,sha256=5Lfpz5C3KDyocZ8qEKncV-mvf192A-3xlMModHwVFi4,752 +jedi/third_party/typeshed/stdlib/2/getopt.pyi,sha256=6hPbDzz4CuSglcyFspFGyWCNVW0AKygfMXPTD0LKI8Q,448 +jedi/third_party/typeshed/stdlib/2/getpass.pyi,sha256=5FZTPudjVwTRUD2BQJhOr0pyxEyTeXUxYmj_xt1VC0E,160 +jedi/third_party/typeshed/stdlib/2/gettext.pyi,sha256=FT_rx4GptZANYzTplDlzUgSQS7aNuiF0V-mYsJzK0eE,2285 +jedi/third_party/typeshed/stdlib/2/glob.pyi,sha256=q5eeP2SFk94tP3PwKZOVOdGhS-XoaGeyXiiKtWVdvwU,375 +jedi/third_party/typeshed/stdlib/2/gzip.pyi,sha256=YIsrQQFHmhyG8XrqIaWZV1glkmXHWa837LAk-f_ZDT0,997 +jedi/third_party/typeshed/stdlib/2/hashlib.pyi,sha256=46pv8pBLAtNJTc2Ndh_IadSJsmzUj_hH8xYzp_cgJFg,971 +jedi/third_party/typeshed/stdlib/2/heapq.pyi,sha256=oKTl_WDSRYheNCenxMjwR0rHAPfAltZ-xLrApwdB1N4,756 +jedi/third_party/typeshed/stdlib/2/htmlentitydefs.pyi,sha256=1dyH0i00daNQ_7gDuT-mxXzx-V_nDSRuF4q_vjkSUHg,114 +jedi/third_party/typeshed/stdlib/2/httplib.pyi,sha256=Pzw93K9nIBJ28c66a-GVmWSiZOaYwtOZB1e6P03adOI,5929 +jedi/third_party/typeshed/stdlib/2/imp.pyi,sha256=8qatIBDkbAIbC137jA5zRPVO9PRgIe8rDUwaceORVXY,1290 +jedi/third_party/typeshed/stdlib/2/importlib.pyi,sha256=N1OsqmcdpbeN7RTNU4s2zQgnoQhkwECT5z5bv035F-g,134 +jedi/third_party/typeshed/stdlib/2/inspect.pyi,sha256=WJqBL9pFUFyIEFEZGvIZOD6mW1AgywTV1UNtTMap-Dc,4692 +jedi/third_party/typeshed/stdlib/2/io.pyi,sha256=_9t3YnEw4vkpHSL39UdKGnOXiqy2L_ps-DZtqK1noPI,1136 +jedi/third_party/typeshed/stdlib/2/itertools.pyi,sha256=2kBOMB5H_2z8-Z-WGWYlSimAiKFNGH1-5inZJJPbe-c,5975 +jedi/third_party/typeshed/stdlib/2/json.pyi,sha256=gouCsXdr9Y4KBmiwsvl6oN8ULrxIAnL-zBiTRIiWGiQ,3206 +jedi/third_party/typeshed/stdlib/2/markupbase.pyi,sha256=GmpHjRP9pa1Ybup4LFHoYS0TKu9oh8EOqX5-CY2yNb4,264 +jedi/third_party/typeshed/stdlib/2/md5.pyi,sha256=RxbpEbnpbF491HuNmiN3legMGS-8W11hHBxE1tcy7b4,74 +jedi/third_party/typeshed/stdlib/2/mimetools.pyi,sha256=vZd4d0QRIrqrvjtei-P-uK-vhduilbUpT55hlBGzIFA,703 +jedi/third_party/typeshed/stdlib/2/multiprocessing/__init__.pyi,sha256=i3uPVztHhIhN7ZQjtUjwbR3qvjZnMAi4qKeE0x5JU5s,1921 +jedi/third_party/typeshed/stdlib/2/multiprocessing/dummy/__init__.pyi,sha256=YQzAQbXIOWxv1gTPvKeNeduNGN2r40OLyPYE7rCo2Vc,1392 +jedi/third_party/typeshed/stdlib/2/multiprocessing/dummy/connection.pyi,sha256=OkvPY8s58y6GC-8yF1y8QyqxZAoZHxeDPfHDOfJXnaQ,673 +jedi/third_party/typeshed/stdlib/2/multiprocessing/pool.pyi,sha256=H24VaPQgs8EFjcmWtI9dn4nvFsjCb7xIDv8xQzTR7Cw,2038 +jedi/third_party/typeshed/stdlib/2/multiprocessing/process.pyi,sha256=2AyzstRaJgHXufF7RSpgBgtVD3BmsdXFgd6SSRwJqKU,906 +jedi/third_party/typeshed/stdlib/2/multiprocessing/util.pyi,sha256=xjwHWnEAqauPnOxaopyvJ6irZXQOv7qZ07PRv0YzCnM,758 +jedi/third_party/typeshed/stdlib/2/mutex.pyi,sha256=lQeSSkY5dwRUOdavqMyCEafd8j5XK6Y76vYMH43tyuE,363 +jedi/third_party/typeshed/stdlib/2/ntpath.pyi,sha256=upbuNccYoJZOEeauXVuegAuTUMr9LsDZ6E0c9PCsJwI,2937 +jedi/third_party/typeshed/stdlib/2/nturl2path.pyi,sha256=_u8yHiGMMnRRTjQAs37HCefvy5193SJDBUZTw1nZ0I4,115 +jedi/third_party/typeshed/stdlib/2/os/__init__.pyi,sha256=njHsqKbPSqEyAtvIOGFdEva0es8A5Siemx_9wu92JZE,13380 +jedi/third_party/typeshed/stdlib/2/os/path.pyi,sha256=upbuNccYoJZOEeauXVuegAuTUMr9LsDZ6E0c9PCsJwI,2937 +jedi/third_party/typeshed/stdlib/2/os2emxpath.pyi,sha256=upbuNccYoJZOEeauXVuegAuTUMr9LsDZ6E0c9PCsJwI,2937 +jedi/third_party/typeshed/stdlib/2/pipes.pyi,sha256=OTfpqql0CUZDbJx-Ka4gWuoDub7FsF7bH7N1PplvU6s,467 +jedi/third_party/typeshed/stdlib/2/platform.pyi,sha256=E8pD_04NuJTBrY5bFIdiAgncffdCfeKjgq_mV_uwIZs,1536 +jedi/third_party/typeshed/stdlib/2/popen2.pyi,sha256=dyNtDrnabvCpQVm7i8TVQpDMIvWKoK_2otJgd41hUYI,999 +jedi/third_party/typeshed/stdlib/2/posix.pyi,sha256=KdclicEcb0wBRDaJhr_avbFQJ3HOYr3U7ZZOyoZjpyo,6396 +jedi/third_party/typeshed/stdlib/2/posixpath.pyi,sha256=upbuNccYoJZOEeauXVuegAuTUMr9LsDZ6E0c9PCsJwI,2937 +jedi/third_party/typeshed/stdlib/2/random.pyi,sha256=MSUCctD3w_mmXrrjtfYKuQ2_DF-26GGNnvrhN8APGZ8,3156 +jedi/third_party/typeshed/stdlib/2/re.pyi,sha256=3pi-S5VeaGgnyZGfY9n0kaBqCR4n7bS5BFjPLt_8e_8,3641 +jedi/third_party/typeshed/stdlib/2/repr.pyi,sha256=Ic8zKDnkZLcpQmOGBLvScWrBx4sYHHmlA4gSoIrSQOM,1095 +jedi/third_party/typeshed/stdlib/2/resource.pyi,sha256=ZMxMAadyqjJ7Nmvu-GFPdy5XwG7RKunEkKu3LuRevPE,877 +jedi/third_party/typeshed/stdlib/2/rfc822.pyi,sha256=PYDCMDz6R87HI7iOVAVbqK4Sr7aTijXVK9kBsDw_KBE,2163 +jedi/third_party/typeshed/stdlib/2/robotparser.pyi,sha256=IpfpnvNmCtN84yyZR9TmNdCQA7F1M5MQcqbUdkwoPXQ,230 +jedi/third_party/typeshed/stdlib/2/runpy.pyi,sha256=D-ttE7Yt0BQGuEMaHf5GUyzWrdW_onB8qwW1Opwrn_E,541 +jedi/third_party/typeshed/stdlib/2/sets.pyi,sha256=hbfbabNHU03vRbZSQ2o1QyghdUm5J5w1eNXpFzHM8D4,2975 +jedi/third_party/typeshed/stdlib/2/sha.pyi,sha256=35pkvQygB0J3E1etUSO_ijjsk_H8Q8k6EYjuNMLFHQQ,236 +jedi/third_party/typeshed/stdlib/2/shelve.pyi,sha256=ac7Jr7saU4x3m9hQIQzPKRhuRJqAQO5_uLvwxcs6u9o,1612 +jedi/third_party/typeshed/stdlib/2/shlex.pyi,sha256=gs-tAXy6la_1_qoR0tQhxfMzXXGWDC3qoir1ofNnOd4,1025 +jedi/third_party/typeshed/stdlib/2/signal.pyi,sha256=qEcMv8og5esD8oDAycqwzU276cqYm6_61n-41UVOJ1c,1571 +jedi/third_party/typeshed/stdlib/2/smtplib.pyi,sha256=8wiSP1iFF9-l9IKgh8p6S0rGwLuguGQfFH3xyWPh4ec,2542 +jedi/third_party/typeshed/stdlib/2/spwd.pyi,sha256=BDoGUDub7DFTKhD_tzXW6DbD3uGX15Ujm2DzuFF_cvA,308 +jedi/third_party/typeshed/stdlib/2/sre_constants.pyi,sha256=T6kBTKeYYGkM83SbbgVx9L38eaZgqEY-AkgfCLr9GbU,1744 +jedi/third_party/typeshed/stdlib/2/sre_parse.pyi,sha256=meZYm_VskCn193qP3aV71kf61HJ8KWYKT55LISLYP9Q,2311 +jedi/third_party/typeshed/stdlib/2/stat.pyi,sha256=Tzy8jDY2wz2pZucTjKwCHly-4C9c3bhLBpQaZW8zk7o,992 +jedi/third_party/typeshed/stdlib/2/string.pyi,sha256=WRgPuXahi1tCgo4iM7HQI7fr4qcc1gOanuuA0Cjh1Qw,3567 +jedi/third_party/typeshed/stdlib/2/stringold.pyi,sha256=cefNX8V9XsaCmguO3e32tYNdfUnfMHZ48Gu8smUUCuw,2010 +jedi/third_party/typeshed/stdlib/2/strop.pyi,sha256=kF2oXemBZd_VaHlTzV19pp9fi8iwcVsq8avQS8YwdXc,1157 +jedi/third_party/typeshed/stdlib/2/subprocess.pyi,sha256=JV6VJpxGqVfwky5ZAmL54wLvYHwEucisA1xWCZunUKA,3282 +jedi/third_party/typeshed/stdlib/2/symbol.pyi,sha256=gmMHvO88vurNaeIXHNnl7UgNPg0gdf8D6iuxX5aTJiM,1341 +jedi/third_party/typeshed/stdlib/2/sys.pyi,sha256=L0K5SKIRsKdL_WJIz735YgaSPvxBXtVPHWY0dvxumZ4,3616 +jedi/third_party/typeshed/stdlib/2/tempfile.pyi,sha256=BbSxin5u5F2m-KXj0ucDGPMIIxyv_lO-VJSpLQa14bY,3696 +jedi/third_party/typeshed/stdlib/2/textwrap.pyi,sha256=8VV3JRR4Lq73UZ4t9gtYOBeM-YcbOCaGbUDPmKHQeJM,1854 +jedi/third_party/typeshed/stdlib/2/thread.pyi,sha256=P3v99RXZZFRPUzpe9St8fswlzP4IEStuPFKdlwlUJvk,920 +jedi/third_party/typeshed/stdlib/2/toaiff.pyi,sha256=FA2QwiYCh-YuKQyMuSj4DhQQo9iTASevNspzkoWfRB4,243 +jedi/third_party/typeshed/stdlib/2/tokenize.pyi,sha256=NC9RJ7aJxIBijxom_sURphAu-rVlnF8j0cHhY_VbJkc,2686 +jedi/third_party/typeshed/stdlib/2/types.pyi,sha256=YKV40Mmm-geFO0fRjHduzNW-0RRYjhOgzoeKWkBVKJo,5465 +jedi/third_party/typeshed/stdlib/2/typing.pyi,sha256=eVRiPuinpIP89d6cVv8AK1x1DlXngv32iRpORi8yNc0,17772 +jedi/third_party/typeshed/stdlib/2/unittest.pyi,sha256=a7yf589r5b0mArYulqLfbvmW8NdJNTnHXZh9rclXTVQ,12726 +jedi/third_party/typeshed/stdlib/2/urllib.pyi,sha256=upPSXvIUWm-CLSW7TwMXageO9i3zNZMpOWzlWR9mmyQ,4765 +jedi/third_party/typeshed/stdlib/2/urllib2.pyi,sha256=xZez6mOO-ku7JSJAEnfZK-Epo0Atd5ZMTfooXukDA6c,8312 +jedi/third_party/typeshed/stdlib/2/urlparse.pyi,sha256=rQEJa_Oyy2KoO9WWgonvFDCfkDgfTgqReuGRU8ky1f8,1944 +jedi/third_party/typeshed/stdlib/2/user.pyi,sha256=Mz3QGfgG58h1uuCR-b9r_Pv8L-vA8oH4I4NN6L6hfC0,83 +jedi/third_party/typeshed/stdlib/2/whichdb.pyi,sha256=k7vfuOkgz6kZd_6mVSsP36DBR16mfRZNpO0kngswZEg,85 +jedi/third_party/typeshed/stdlib/2/xmlrpclib.pyi,sha256=6H5putpgG72kUD6x5OLeJKtMzXkvrMMOkNx-zShk27U,9769 +jedi/third_party/typeshed/stdlib/2and3/__future__.pyi,sha256=b7dmNhiJdcJM2cyLfX1i73MNfiVwTUfMmyOdZzBU2fw,587 +jedi/third_party/typeshed/stdlib/2and3/_bisect.pyi,sha256=MURhFGAf_n_yms64euXpOGBeEY3932tibeH9gkxDsQc,471 +jedi/third_party/typeshed/stdlib/2and3/_codecs.pyi,sha256=i_rEwDVsWLSSK7M1ShCr3jZ79V9MwQY4on1CF8PBQ74,5308 +jedi/third_party/typeshed/stdlib/2and3/_csv.pyi,sha256=OZpwhCxRKWUNfzD-TFlmjGgx1WnbSAY1f5CruGAieEc,1574 +jedi/third_party/typeshed/stdlib/2and3/_curses.pyi,sha256=nI5pvFuJzbmoa8ZOIZ4I94UPStHtWM_wQsO84a8J4pw,13596 +jedi/third_party/typeshed/stdlib/2and3/_dummy_threading.pyi,sha256=__eLfXz1Hygl5swKoIL4JZtQ-JBIKkr-yExKwKl6eD0,6309 +jedi/third_party/typeshed/stdlib/2and3/_heapq.pyi,sha256=heTQYQlAQ5f-78bm3rPkSeoQWoxIQ4_pX44TcFx0AGE,612 +jedi/third_party/typeshed/stdlib/2and3/_msi.pyi,sha256=tAOc34He8wJWmW1WsINZxT0ns-a8zYfV-rlhnDKzg-g,2168 +jedi/third_party/typeshed/stdlib/2and3/_random.pyi,sha256=Ot_yeMIsjSxwYFMkes_URP2OlZqzRePVeYaCbnQr-qI,478 +jedi/third_party/typeshed/stdlib/2and3/_typeshed/__init__.pyi,sha256=auqNqMA9bWUD9jKmsHaWidPPnFFcc0M4c4a9M3nHChU,4615 +jedi/third_party/typeshed/stdlib/2and3/_typeshed/wsgi.pyi,sha256=n0p7egUjhY7Hni-BwmOZd1OpOWy_GkpNDKDYHHHcrKE,1293 +jedi/third_party/typeshed/stdlib/2and3/_typeshed/xml.pyi,sha256=sWVM_2a0LZudB2irKi_UO4fJHhMiYz8OFLeTtvhByz8,528 +jedi/third_party/typeshed/stdlib/2and3/_warnings.pyi,sha256=5h2niNh47xb7G7p6MOvQSj-NVnFJf_hiwyO4aLN05BY,2233 +jedi/third_party/typeshed/stdlib/2and3/_weakref.pyi,sha256=-638OW7gOKVYT51Bp8QkXKc13qvnhpl_6NCQCvdaY6s,1200 +jedi/third_party/typeshed/stdlib/2and3/_weakrefset.pyi,sha256=FY1Fl7JZpGeVBFKZV3MhCGOR8WoD8jGnH6g7Gd_0OW4,2417 +jedi/third_party/typeshed/stdlib/2and3/aifc.pyi,sha256=R9iiBcPS2xH4RSRcwydKcpWuxhDP4K4HANvaYLsERKk,3393 +jedi/third_party/typeshed/stdlib/2and3/antigravity.pyi,sha256=hXe2_7W39SOiBPpQJb8e-9pre8_0DIGWXRZY1IFp-6c,123 +jedi/third_party/typeshed/stdlib/2and3/argparse.pyi,sha256=7AG5GfwwiCJe3UqjaIRP5xtbyRNKCyqOunne245uml0,18335 +jedi/third_party/typeshed/stdlib/2and3/array.pyi,sha256=VADSEevIbSyboD2_W-2eF51q8nJp2QcgPF5d0Siqp_c,3489 +jedi/third_party/typeshed/stdlib/2and3/asynchat.pyi,sha256=v-upDoOfXdG1839UdJj6wkBosW0914umWN-BOjaGrwM,1555 +jedi/third_party/typeshed/stdlib/2and3/asyncore.pyi,sha256=OS5PpOKzhv4BM7wNNTDgky5rJtoFE1yuz-ZIZ7bPwdc,5534 +jedi/third_party/typeshed/stdlib/2and3/audioop.pyi,sha256=lBRWVdim8hPgzNwMIWQSs_nvv7b7uoBqjuEbXG7FiqM,2119 +jedi/third_party/typeshed/stdlib/2and3/base64.pyi,sha256=E8MqdBGhXIjIXkqSkLdkS6645St1pMYR59syRKr5oeQ,1614 +jedi/third_party/typeshed/stdlib/2and3/bdb.pyi,sha256=dZP-An_qNDJXxMzWsIX1fcTIoc3qI-WIm48BS3gAJLk,4644 +jedi/third_party/typeshed/stdlib/2and3/binascii.pyi,sha256=HgN_QgHsAdhRgpelHyZsSWnup-Tce23veb46498Mo6U,1552 +jedi/third_party/typeshed/stdlib/2and3/binhex.pyi,sha256=KPO4jyuFjgvRV1ryvanL0j0cIThoIoDisYOz3TBm_nw,1147 +jedi/third_party/typeshed/stdlib/2and3/bisect.pyi,sha256=sQn9UUS0Cw5XZMEGcEj8Ka5VKPVobL43Pex_SagjXg8,67 +jedi/third_party/typeshed/stdlib/2and3/bz2.pyi,sha256=RO-rYMH1TpprITXeCtqhU5TM6I7gQDRutpgh_bSvZAo,2267 +jedi/third_party/typeshed/stdlib/2and3/cProfile.pyi,sha256=IhoIaaoA9_bIN4ifc6d0qX3kO1so5Lss17OCrBW8HcE,1239 +jedi/third_party/typeshed/stdlib/2and3/calendar.pyi,sha256=iB08ixpoedGoLTBZovdy7JavZEqFYF0kxwXctgez0qI,5773 +jedi/third_party/typeshed/stdlib/2and3/cgi.pyi,sha256=HUlFXoovlydtwen1RnQdlLPQC4a26fXe5wrAR0f38bM,6016 +jedi/third_party/typeshed/stdlib/2and3/cgitb.pyi,sha256=-5_uqith9S9ZHOXQf-2Z76Q3I7UeBX2kxRoijoQbvnI,1447 +jedi/third_party/typeshed/stdlib/2and3/chunk.pyi,sha256=juLgoUjtZJj3tExr1grorsVNUnVZtTgoJ8N9gj8opXs,613 +jedi/third_party/typeshed/stdlib/2and3/cmath.pyi,sha256=jSud27iMIBSTw2I2NhIUJa38VEC8X1h7YlN7SbQqWhk,1217 +jedi/third_party/typeshed/stdlib/2and3/cmd.pyi,sha256=-8dq1Td5QhOzLoBkeK3EZjYTfNvxJUTZHrBj0H03cl8,1658 +jedi/third_party/typeshed/stdlib/2and3/code.pyi,sha256=A9cTESdR37TyuZeiiIjfDJhI-0Baizcu_YGHZU6hqTU,1522 +jedi/third_party/typeshed/stdlib/2and3/codecs.pyi,sha256=DXqiWmZp5ZhfIw6zib357lvj9OnTmMmKgh8uQ6SBLu8,12312 +jedi/third_party/typeshed/stdlib/2and3/codeop.pyi,sha256=Ypyu2k5z5r6B2tNkO9JKM8YM-CBXT3SZUL0Ak8P8qag,489 +jedi/third_party/typeshed/stdlib/2and3/colorsys.pyi,sha256=JwCA0W2O9-JS0KA9u7kKNVJKRiu1ACOPzxy-SEKzDRg,578 +jedi/third_party/typeshed/stdlib/2and3/contextlib.pyi,sha256=sV5PoppaY5WggvWnKN7ZpAnKWFITWtTPV6CEPLnmrKY,4419 +jedi/third_party/typeshed/stdlib/2and3/copy.pyi,sha256=EOWzpARbTnxmkq3u6Om9nYpYG62ts2WqfquBObFQgxM,328 +jedi/third_party/typeshed/stdlib/2and3/crypt.pyi,sha256=_D__DDlGumti-2YksSJiOjFJ0tJbckFBKrszPdbtWrg,648 +jedi/third_party/typeshed/stdlib/2and3/csv.pyi,sha256=26HmvwJBnGEWObBRHj0xS0p68q1192_ygtQd2rEgB9g,2751 +jedi/third_party/typeshed/stdlib/2and3/ctypes/__init__.pyi,sha256=Br0DywqyGaVwHaWbgbIWZLnQYQ7Spa04Z4eU8TrHz-8,11893 +jedi/third_party/typeshed/stdlib/2and3/ctypes/util.pyi,sha256=fO9_MTBB4lpXUvuQzrFT4yuIzt9x_FsEWtW6kbAWtkg,163 +jedi/third_party/typeshed/stdlib/2and3/ctypes/wintypes.pyi,sha256=4mLfzJ8kXytQo4DDsO5HX13sZWXUcs-XdwPygO6MOE0,4642 +jedi/third_party/typeshed/stdlib/2and3/curses/__init__.pyi,sha256=e6zyQJFe4QJgc6mUK3ZqOqRTKGXq_QxJLXLs3vyVQHU,370 +jedi/third_party/typeshed/stdlib/2and3/curses/ascii.pyi,sha256=9MeRhsGJwmIcMsL77Ct9QOrt5TKS_JRtgjyv_nbmQZQ,1212 +jedi/third_party/typeshed/stdlib/2and3/curses/panel.pyi,sha256=Wsl42xkXk8GQesNABDijIoBVX5Nx8dGm6prO1-gxlyU,801 +jedi/third_party/typeshed/stdlib/2and3/curses/textpad.pyi,sha256=V-6r4xPbkITORxiUCAPV-QzWi69JZV0tZwO72HDbuU8,457 +jedi/third_party/typeshed/stdlib/2and3/datetime.pyi,sha256=vfY8l1ocVFixSnpqCIp6nGbp8NT1aU_hksSYpeAoTpw,12699 +jedi/third_party/typeshed/stdlib/2and3/decimal.pyi,sha256=94S0G8TK1ytezWJuoH_gHpnAeNswravx_BXo3nJO9Mc,17787 +jedi/third_party/typeshed/stdlib/2and3/difflib.pyi,sha256=gZTDgj3DFNFmIpbQ_Ju2zcsVT3xop8yB1JSbAi1XWz8,4936 +jedi/third_party/typeshed/stdlib/2and3/dis.pyi,sha256=3FFtM_e7jcMgzXw60pTLZZWMEO4nV3K6iZVuXwbSwXQ,3089 +jedi/third_party/typeshed/stdlib/2and3/doctest.pyi,sha256=kfWLoM--CjzB1TW3481RvTfVDOOgbQy1oL94-2joI2M,7083 +jedi/third_party/typeshed/stdlib/2and3/dummy_threading.pyi,sha256=ZI04ySfGgI8qdlogWtA8USUTFGfzm32t2ZxL5Ps53O8,79 +jedi/third_party/typeshed/stdlib/2and3/ensurepip/__init__.pyi,sha256=dTpEX_z-rFEqnQI3XvpxGCw-IE1aqUxZw6KEv0k86So,562 +jedi/third_party/typeshed/stdlib/2and3/errno.pyi,sha256=KDjlJTkt1sdcrwGgLkPMZlSwz1Dp0Xkt6PqEnqcLZWY,2011 +jedi/third_party/typeshed/stdlib/2and3/filecmp.pyi,sha256=z5T2uxVBjwdWIzrhR-zlRqIV9rg08EpUldQgr3-AfJ0,2566 +jedi/third_party/typeshed/stdlib/2and3/fileinput.pyi,sha256=01vjE75hR9lXLH-S_MN-RAeieTiBGcg4_of-zNyDmnU,2601 +jedi/third_party/typeshed/stdlib/2and3/formatter.pyi,sha256=lxpoZgWeuh3fyJu7ew4O00DBospTKRxateEK7f99zMU,4639 +jedi/third_party/typeshed/stdlib/2and3/fractions.pyi,sha256=VblZe5mtSJGL7fUSRronuksTeO24OLVAGghS_rEDpog,5936 +jedi/third_party/typeshed/stdlib/2and3/ftplib.pyi,sha256=4FqgC8unm_zR1CcpJalpHvRzODOmy0IuZk_Fggoercc,6278 +jedi/third_party/typeshed/stdlib/2and3/genericpath.pyi,sha256=Wob25JMIB38kmxD3fbyEmW_zOZ30A-0RR41OymvOg_E,806 +jedi/third_party/typeshed/stdlib/2and3/grp.pyi,sha256=fm0JOwvJ8CGCXSVWyqQuXFBPBfoVC8b5CyWTq-5fcvs,295 +jedi/third_party/typeshed/stdlib/2and3/hmac.pyi,sha256=DRSe8IPnvSQ6DCIhG8IFosACcpsOngQftNJRjEj4WkU,1607 +jedi/third_party/typeshed/stdlib/2and3/imaplib.pyi,sha256=d8rIKtUeqXt5O51dNweLcp_J6445KdC9aWJnGKR7zdI,8034 +jedi/third_party/typeshed/stdlib/2and3/imghdr.pyi,sha256=38fd4eD_xncwov0L7vvbaDGjKi9gGzLPbgsD3oe_I0A,604 +jedi/third_party/typeshed/stdlib/2and3/keyword.pyi,sha256=48hMnou7HbA580mGfRyd4uPgxmnbSt4KM_STSGMFzFA,210 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/driver.pyi,sha256=FqgZDUSf1X9X55R7dm4YxYIIzu3zAXjB0x6T_zQl92w,956 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/grammar.pyi,sha256=zuKtRoL1GMizL8ij0akjhZVXH2mBsbb2LxJkFJPi0ME,733 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/literals.pyi,sha256=CSUI0I_W6I9fN-nb6rGgzLjGMIO0C6OwOmzYz-HEQZU,172 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/parse.pyi,sha256=Vd7LRmWw5Q9pWzYBbDsKAvL2NdyCZNEFCa0kysAtrsU,1107 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/pgen.pyi,sha256=ZhxquDht1cGiy0KiclTZSKGSctTRM21QTQDJEY38nC4,2140 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/token.pyi,sha256=YpBo_itTcpSXfWPpPFYEdTLcdEerPoAw8nuHpAGEo1Y,1065 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pgen2/tokenize.pyi,sha256=pA9uvZ22MHGVm37U1xhMEde2hUPvJQJBq6XDQ3Gzcts,883 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pygram.pyi,sha256=lyhgf6kpNVMPN6g6W-DSxt8E2_IyVguf7JjKzVhAmzc,2208 +jedi/third_party/typeshed/stdlib/2and3/lib2to3/pytree.pyi,sha256=Zz0sUjSMFvxHKpIPlvzNCRyJtzT2vEzYKPPGha8LRFk,3322 +jedi/third_party/typeshed/stdlib/2and3/linecache.pyi,sha256=mfzNP8AUc3yd_VahaBEaSYt7-zDAKneFa7rjFQE8sIU,591 +jedi/third_party/typeshed/stdlib/2and3/locale.pyi,sha256=t-y1NnMHom3HdCewxj1zq7-83xAEdC2FR7KC7P3K67w,2555 +jedi/third_party/typeshed/stdlib/2and3/logging/__init__.pyi,sha256=zMbblZS9jbaC4hrktbUu0rzfZKP_AKhN2shNU0kgh_M,28358 +jedi/third_party/typeshed/stdlib/2and3/logging/config.pyi,sha256=oreN5ILtTK6x7s5m0n7-wqM4fG3e4WYp8uRhu9Qmtl0,967 +jedi/third_party/typeshed/stdlib/2and3/logging/handlers.pyi,sha256=J36V-7X3w-tFAH57rE6YkYU20tvEDzQF_omTvHIPDp4,8847 +jedi/third_party/typeshed/stdlib/2and3/macpath.pyi,sha256=B6y98iSZAXQPkStHypSps3hvpzq4eBmfYL6uMyBYj4E,5330 +jedi/third_party/typeshed/stdlib/2and3/mailbox.pyi,sha256=IrYfGXuaCVIJDADHUZ-1D5OOvHHeULJdTsWySzYCL5I,8071 +jedi/third_party/typeshed/stdlib/2and3/mailcap.pyi,sha256=5MWzikKoMQEkohWxGbzZUPwY6E_g1-i-nvPQw13Zxwk,330 +jedi/third_party/typeshed/stdlib/2and3/marshal.pyi,sha256=zXZaX_H3VD9clK-iiNZz7f5GKDdG4_TriqlTPCBR6oM,253 +jedi/third_party/typeshed/stdlib/2and3/math.pyi,sha256=RA890_35QPnZF7K2393I3qhr0My1U62LVK2UjXyv9Fw,3835 +jedi/third_party/typeshed/stdlib/2and3/mimetypes.pyi,sha256=wEC12QuS8DXgpM9ZMB-SZ1Zh2GA494p-cRBZQvnV6dI,1641 +jedi/third_party/typeshed/stdlib/2and3/mmap.pyi,sha256=BZHJGHPpkicsCQSIgZOUZA62JaXkafcpm4Vj8cYZjsU,4006 +jedi/third_party/typeshed/stdlib/2and3/modulefinder.pyi,sha256=Pwx90XH7YRRF4Dm1mUNBtIP8xb-5eDwTi6bwl1x4k_o,3674 +jedi/third_party/typeshed/stdlib/2and3/msilib/__init__.pyi,sha256=vruWL_rWAZiYVXQMq4jwiqVTzrIXKw2g5PSf59FOJu4,6306 +jedi/third_party/typeshed/stdlib/2and3/msilib/schema.pyi,sha256=LEZZXliX-2N73ryhJtTAuTvP_RZVsD4i7S1CRTLNtik,2214 +jedi/third_party/typeshed/stdlib/2and3/msilib/sequence.pyi,sha256=XQC-7D28bgEuJzAFC7WIX86AtUBrQ7pC5Sc1JTyPwEk,356 +jedi/third_party/typeshed/stdlib/2and3/msilib/text.pyi,sha256=drG3KVxFc8MYRrdkm3oJWaivBvZecq6pdKoFEemvF2Y,202 +jedi/third_party/typeshed/stdlib/2and3/msvcrt.pyi,sha256=DDiTa2YmqkChpLzKr80n-ZOfJAXWb6YYB0RER013dHw,795 +jedi/third_party/typeshed/stdlib/2and3/netrc.pyi,sha256=6ipkr1k-p5Pve7y-PTtCH0sZqeFV19Aw33SkQddMJm0,444 +jedi/third_party/typeshed/stdlib/2and3/nis.pyi,sha256=CUnTx-mKL-YinbUfrvw8WIWLwuqu4PtSVZ9M2mxSSvY,322 +jedi/third_party/typeshed/stdlib/2and3/numbers.pyi,sha256=73gLvE-jPZrLxYYhVHJfA--xYyIjJJG9_06iU3Vdauk,4335 +jedi/third_party/typeshed/stdlib/2and3/opcode.pyi,sha256=k_DFbtjvSdVKzq72SSztF8K4GKK-tk_WMSITs6EKUTI,609 +jedi/third_party/typeshed/stdlib/2and3/operator.pyi,sha256=JoI0OeQjkMbyEoD4cYXn9R1CVf2GXNOlFKKkgKfUmZ0,7146 +jedi/third_party/typeshed/stdlib/2and3/optparse.pyi,sha256=6YXlTrzWD4rA0HUYfQ8odZlsCza7ln4bT9srOGv7d-k,10096 +jedi/third_party/typeshed/stdlib/2and3/parser.pyi,sha256=AZjxkjNl47Rk3MMk38Muh7gpibhfs6EeTrHNubEsgTA,966 +jedi/third_party/typeshed/stdlib/2and3/pdb.pyi,sha256=mppJ9PSllcPnOcziqt3ua1Ne-N8BKDC7_kCa0xCNc7o,10325 +jedi/third_party/typeshed/stdlib/2and3/pickle.pyi,sha256=HgJusFb0xpfdwPMgh689OtA9zWULEUC_ob_3bSQNVgk,5322 +jedi/third_party/typeshed/stdlib/2and3/pickletools.pyi,sha256=LekbII856jROKj6nerzKVY3tr0J3rjFh7aHiiB9fMHQ,4510 +jedi/third_party/typeshed/stdlib/2and3/pkgutil.pyi,sha256=EXApPfIH6cq6nCJ3dgWY27Z2vO95y0_4sjUspTLJRSA,1558 +jedi/third_party/typeshed/stdlib/2and3/plistlib.pyi,sha256=SufCK1SjP-QOhGd1VhStpLcdRiuoczBGX3KdB4C4ICs,2742 +jedi/third_party/typeshed/stdlib/2and3/poplib.pyi,sha256=CezxOiUqIikxqAv4df5G5rs2jqmGMIWYVfbYt2hvUZg,2500 +jedi/third_party/typeshed/stdlib/2and3/pprint.pyi,sha256=Eox19Bw2R4Uyl87ilwJHr38pEFhBdHRWC2TgW9A2T6E,2893 +jedi/third_party/typeshed/stdlib/2and3/profile.pyi,sha256=4mezNO66Y3vWX9SIftkcbeQcA3Swuc62yNxPkmmwk1w,1199 +jedi/third_party/typeshed/stdlib/2and3/pstats.pyi,sha256=trO8Oc7GQ4nRAdays5pWi5Viib1tKQwkwnQH0upmcJ4,2213 +jedi/third_party/typeshed/stdlib/2and3/pty.pyi,sha256=aCa6VBas1vNafTUiBCmFI-W9SR018bDjy1WtJmIqxOg,592 +jedi/third_party/typeshed/stdlib/2and3/pwd.pyi,sha256=s48Hp0pbMoMBpWg6glXl1ABEcLU4zLj_wV-PloenVI0,350 +jedi/third_party/typeshed/stdlib/2and3/py_compile.pyi,sha256=1A3_VdW5WwbifT-CTrSoXy0ZNFoN5zsqUUuUyPXJjfA,1642 +jedi/third_party/typeshed/stdlib/2and3/pyclbr.pyi,sha256=xirCZDC7Q7ffw0CZ3V6_62EbwCSSE4UXe0-DYatTLUs,1192 +jedi/third_party/typeshed/stdlib/2and3/pydoc.pyi,sha256=aHbKeX9juIvggS2xrKYVGkVBHvxJRBAJLMmtuYS5Xms,10660 +jedi/third_party/typeshed/stdlib/2and3/pydoc_data/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2and3/pydoc_data/topics.pyi,sha256=PLI9r9ujTHeIKw7D9GxCeB_LwxV17eUTauVx9AKiU8k,48 +jedi/third_party/typeshed/stdlib/2and3/pyexpat/__init__.pyi,sha256=XJx933QTavu4mJNP63uHUFwncDx2wlbv1O1DarVJQ9E,3404 +jedi/third_party/typeshed/stdlib/2and3/pyexpat/errors.pyi,sha256=TVdXkdX10ZCjPuBYAy3yNQzUJMo5lCjsl8bv4VJ8PU0,1275 +jedi/third_party/typeshed/stdlib/2and3/pyexpat/model.pyi,sha256=LlBMaLvt3TBD1JQSAQTYUrHKAfKNEeQYaDw5GsJA--g,205 +jedi/third_party/typeshed/stdlib/2and3/quopri.pyi,sha256=0zH--hEadPU_f_GNqwlxU8vrs2tqmPaLCO94BeZXCyo,343 +jedi/third_party/typeshed/stdlib/2and3/readline.pyi,sha256=-Yx9NN7kGQwlGTwubEhIUIFPiMHQJY0z8QbuCE8u7xo,1585 +jedi/third_party/typeshed/stdlib/2and3/rlcompleter.pyi,sha256=1aZLE2z6iAikB08PpAEuc-UeL63VZhgHYQpBT4ssVGI,308 +jedi/third_party/typeshed/stdlib/2and3/sched.pyi,sha256=D7Y2BV6CNyRrJNn0UwLp1sYVTxu1TMw_CR4U3HUs03Q,1497 +jedi/third_party/typeshed/stdlib/2and3/select.pyi,sha256=zpT6GIDgll2LngtzJbW2ZV1qLyJAMpLwJwLMKynskxI,4828 +jedi/third_party/typeshed/stdlib/2and3/shutil.pyi,sha256=O5GZv_Ion6daXUfCfJLjhNPfefuLArzzCK0cYWVVxuA,5961 +jedi/third_party/typeshed/stdlib/2and3/site.pyi,sha256=dG90RROJAZP04P8KTzbJOEm5w-5lbHPfP1tU5IbErJo,450 +jedi/third_party/typeshed/stdlib/2and3/smtpd.pyi,sha256=mKs96L0YHWx4Mk3DXIq0ROLiettU1muAHz_MoksxOwk,2935 +jedi/third_party/typeshed/stdlib/2and3/sndhdr.pyi,sha256=Kbdxb_xqFGOZzce1apkDVbj2dpoMWKplNAP5KNV6tRc,501 +jedi/third_party/typeshed/stdlib/2and3/socket.pyi,sha256=y2Db0P_TYtBdVuWiG3xXxsr0AbSxa5sXbyLf0or_wuI,22711 +jedi/third_party/typeshed/stdlib/2and3/sqlite3/__init__.pyi,sha256=aJu9MCNl8y9HCykdUpo1Z-LSiP3mxRSxrWkCsMxemYI,43 +jedi/third_party/typeshed/stdlib/2and3/sqlite3/dbapi2.pyi,sha256=hdn9VtF3aSsI-oGdY4XObtCo8kSUVGOPuZP51f2Z9aM,11315 +jedi/third_party/typeshed/stdlib/2and3/sre_compile.pyi,sha256=a7wV8OxkRzj0F1L7B02gcZrvMi5Uz87g2wG0pHvWtNI,1073 +jedi/third_party/typeshed/stdlib/2and3/ssl.pyi,sha256=igBbyUhdonNcvRkjzUaRf2r8LIOx1J2lwYzHVC9AEvk,14626 +jedi/third_party/typeshed/stdlib/2and3/stringprep.pyi,sha256=fqKAxHgpLExMmFDO66rJ-kFZS5mVKeWvK_qWQ2yvsWc,817 +jedi/third_party/typeshed/stdlib/2and3/struct.pyi,sha256=FnlORamrhrWi59qMGSXUL_px20rH8QAgFyrptIv3Qxw,1568 +jedi/third_party/typeshed/stdlib/2and3/sunau.pyi,sha256=8nruVjfJtTIU816plhmHsGL5VRubbsWem5hvWiyiONc,3085 +jedi/third_party/typeshed/stdlib/2and3/symtable.pyi,sha256=Dff6o3EDAli4Mzo-Jx8aSN9iiCxlEN-hC82OFaJOHVg,1645 +jedi/third_party/typeshed/stdlib/2and3/sysconfig.pyi,sha256=ISugdNUmN9ycQF8mHoiBL8zoy9As_QcIBRvRPbUTamY,843 +jedi/third_party/typeshed/stdlib/2and3/syslog.pyi,sha256=WbINKcwKTg5WFsn4f06K3-jriyoYXzqnOU9HwbAFbZY,821 +jedi/third_party/typeshed/stdlib/2and3/tabnanny.pyi,sha256=g_vgZl-PEUiu0JEx0LxnwPaDmgP65k9Hva0zVCIDUQU,447 +jedi/third_party/typeshed/stdlib/2and3/tarfile.pyi,sha256=1RRfYoQGC8KVZABw5FH0PGk5I3cgS90HWN_s-EoboRQ,8182 +jedi/third_party/typeshed/stdlib/2and3/telnetlib.pyi,sha256=TRsxrafitwtsxZ6m-KzsFEy-9uFpypw-r2bmW6Mz2lQ,2680 +jedi/third_party/typeshed/stdlib/2and3/termios.pyi,sha256=ZJKwv-a6e8w8frrD3Wgo_I66YNXNtQkhY59jymFZ-wI,3504 +jedi/third_party/typeshed/stdlib/2and3/this.pyi,sha256=5hi7CD2tChI9_genuMBxsS8GOFo0gAVvFGuv-_Uc9p0,50 +jedi/third_party/typeshed/stdlib/2and3/threading.pyi,sha256=__eLfXz1Hygl5swKoIL4JZtQ-JBIKkr-yExKwKl6eD0,6309 +jedi/third_party/typeshed/stdlib/2and3/time.pyi,sha256=jnR64tSQq-k6AOYxmFOxLD2FPFy71v0BPWsic4F1fVM,3900 +jedi/third_party/typeshed/stdlib/2and3/timeit.pyi,sha256=AXtKdk4M2pfi6_KCgmpkDujbaVuR1smNIgpz6_90fIQ,1623 +jedi/third_party/typeshed/stdlib/2and3/token.pyi,sha256=04p49poqkYTWUiSpb2syf7dX84i-eP_NBUXbJIkFcRY,1468 +jedi/third_party/typeshed/stdlib/2and3/trace.pyi,sha256=JzpexVtFU46sf7VlGb7mnpvyLMKxuco6OAk-pQRnhLM,2037 +jedi/third_party/typeshed/stdlib/2and3/traceback.pyi,sha256=O9stbkIrGRLbmSg-nBxrEFlEkk9Kxp-yJiQXnJzc5hs,5545 +jedi/third_party/typeshed/stdlib/2and3/tty.pyi,sha256=A25_a1yrTL55nQAsEpOWKsni215-75a4rAoFJ7g7Qr0,275 +jedi/third_party/typeshed/stdlib/2and3/turtle.pyi,sha256=sbaGlYhQbjQoqfJNX_mxNqMeJEqjNpuVQJe5L0Ci4Zs,19476 +jedi/third_party/typeshed/stdlib/2and3/unicodedata.pyi,sha256=AdW8_ww_4_Oh5undlAXiNUJF6elKYALIyc_cMMJsqpg,1902 +jedi/third_party/typeshed/stdlib/2and3/uu.pyi,sha256=j8lHTLVoMNPv2EJ1py5ozflgE5M1tHFrCOjydfSbPts,549 +jedi/third_party/typeshed/stdlib/2and3/uuid.pyi,sha256=NnP8AHc5cneMi37IHGdBBvmVqPSAlTUK_4qOfEYMMhA,3448 +jedi/third_party/typeshed/stdlib/2and3/warnings.pyi,sha256=byrds4TJ9vuk0d5ohkeFcm81yadHPfmudeFBE1JEQ8A,2583 +jedi/third_party/typeshed/stdlib/2and3/wave.pyi,sha256=EWn5CUj5JmmpZHDy6nfHuxkLSHhEZ8vPl-iH5rOeyrk,2651 +jedi/third_party/typeshed/stdlib/2and3/weakref.pyi,sha256=yK1h8AhDi0SMSMS3jih101FhTimqaZl_g-uIcc1hcNQ,4433 +jedi/third_party/typeshed/stdlib/2and3/webbrowser.pyi,sha256=Aic7NcY6wXkRP5SU66vAhrcWylwe8i78u2nzfenn7FY,3295 +jedi/third_party/typeshed/stdlib/2and3/winsound.pyi,sha256=i7oaOMrdRL5hMtBGZXQtB2jR_gIDcqltc0JMZPeTHMY,811 +jedi/third_party/typeshed/stdlib/2and3/wsgiref/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2and3/wsgiref/handlers.pyi,sha256=HkQCK0geDJd1kFX6FWlydyu7QWG-rcDM2P2BHXKAsrI,3125 +jedi/third_party/typeshed/stdlib/2and3/wsgiref/headers.pyi,sha256=j5-Y2n64U9US_Xld0JtgrWjA8eYlo6BOzu89HKdDsaE,1250 +jedi/third_party/typeshed/stdlib/2and3/wsgiref/simple_server.pyi,sha256=4pAabsnp0x4uyrHdtxchezvI8cudyl4F6K4VlCX5VFk,1528 +jedi/third_party/typeshed/stdlib/2and3/wsgiref/types.pyi,sha256=1qhS0qVWoV0IVfe3b1y4Mzhou65EPtwb_QLmWfX5_I4,71 +jedi/third_party/typeshed/stdlib/2and3/wsgiref/util.pyi,sha256=Ly4ctt2LOpbP9t5YeKU9wwov7iN1uFqeAtBjJ-O1HtU,893 +jedi/third_party/typeshed/stdlib/2and3/wsgiref/validate.pyi,sha256=YB_yhIz9Dl3b8s_-ASpzsPpRSpCwCuZ1jMaLAB_7q4g,1861 +jedi/third_party/typeshed/stdlib/2and3/xdrlib.pyi,sha256=yHZnDsb7h-kuMZ04mQip7YRjBSX-g900SOE0HhGyMl4,2315 +jedi/third_party/typeshed/stdlib/2and3/xml/__init__.pyi,sha256=BqMXnsXiYPoalMzEakn6mYDxgyW5N2UPF0Ao7xPuGVY,30 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/NodeFilter.pyi,sha256=bi0L5SEOxk4FyEhf18oU-I8Msf9S9o_tJt-mVc93f28,457 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/__init__.pyi,sha256=gjfWhkwyNoY8SeH6cztWZ9W8w9E4CLgCpHeP8vnHM5c,1844 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/domreg.pyi,sha256=Sq02GZ6VRiXWCy2lUE1e47_EUDdr88rmWZoKCxH0fgQ,462 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/expatbuilder.pyi,sha256=wI_eu1G8yaaquRHmZ9mYRgjy4zNNhJC385TjSMoamRg,77 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/minicompat.pyi,sha256=wI_eu1G8yaaquRHmZ9mYRgjy4zNNhJC385TjSMoamRg,77 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/minidom.pyi,sha256=DSNxhSfqdzlQQ5HjeRZHxB2mhshhLkDfeAmeOY5I9K4,287 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/pulldom.pyi,sha256=wI_eu1G8yaaquRHmZ9mYRgjy4zNNhJC385TjSMoamRg,77 +jedi/third_party/typeshed/stdlib/2and3/xml/dom/xmlbuilder.pyi,sha256=wI_eu1G8yaaquRHmZ9mYRgjy4zNNhJC385TjSMoamRg,77 +jedi/third_party/typeshed/stdlib/2and3/xml/etree/ElementInclude.pyi,sha256=4e6cquTvRpyBS1UBTNomH5ghEtSzMHr3im5VwFAzvEI,873 +jedi/third_party/typeshed/stdlib/2and3/xml/etree/ElementPath.pyi,sha256=lJV3KlSWuf_anuM3hIRwdD-n-hpO4b7MzCKg0FxyFAE,1561 +jedi/third_party/typeshed/stdlib/2and3/xml/etree/ElementTree.pyi,sha256=YbNf8vOL8aKD-MjSJufTgeTcip6A270muX9zWiKQzDQ,14905 +jedi/third_party/typeshed/stdlib/2and3/xml/etree/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/2and3/xml/etree/cElementTree.pyi,sha256=D25RVU5Y1Sai47EQ49UTStrYaY39HMqT1HUOZrSioRg,50 +jedi/third_party/typeshed/stdlib/2and3/xml/parsers/__init__.pyi,sha256=FHZYB9bXDrj4RiKUgrctpkuf7_Rms9PqQrGyjkn0EE4,34 +jedi/third_party/typeshed/stdlib/2and3/xml/parsers/expat/__init__.pyi,sha256=qmz8tuPGbZ2rBfRrfYANxDZNxn9BTQXdd9AugF5wDW0,22 +jedi/third_party/typeshed/stdlib/2and3/xml/parsers/expat/errors.pyi,sha256=mH9YRZuV4quzksDMLEmxiisAFgNhMOhl8p07ZzlS2XE,29 +jedi/third_party/typeshed/stdlib/2and3/xml/parsers/expat/model.pyi,sha256=M7GVdd-AxOh6oGw6zfONEATLMsxAIYW2y9kROXnn-Zg,28 +jedi/third_party/typeshed/stdlib/2and3/xml/sax/__init__.pyi,sha256=PuxAYEbGDZ-GvZFBPhpKAnN5935WR5dRWaGbhdUN1Nc,1389 +jedi/third_party/typeshed/stdlib/2and3/xml/sax/handler.pyi,sha256=lzRvfmcsC4Px11th3C-OB53OJgrSxHHTkgWKkFnYYII,1391 +jedi/third_party/typeshed/stdlib/2and3/xml/sax/saxutils.pyi,sha256=GPHCPJVekBwXRuRViACPSwgJ6WvTn3bKK7Vfy4hgFwA,2825 +jedi/third_party/typeshed/stdlib/2and3/xml/sax/xmlreader.pyi,sha256=6xszLDpI9jWetj83aFhYiq2XKeTvp6A1vj0gT2grXqE,2477 +jedi/third_party/typeshed/stdlib/2and3/zipfile.pyi,sha256=WDrnBqQDKJpsKwqAATU8DrbT4_9fi1ZtMz_6j1antOQ,7249 +jedi/third_party/typeshed/stdlib/2and3/zipimport.pyi,sha256=zdgTOt05qDHBEdkU8TmZXXu_VeX4BXptVsGnYu39eiM,1244 +jedi/third_party/typeshed/stdlib/2and3/zlib.pyi,sha256=fn9NyYniW7bGyLOaipjK0mBXQnUqlmowBBc5BPAM_gk,1692 +jedi/third_party/typeshed/stdlib/3.7/_py_abc.pyi,sha256=vReNkdPnizulbWOKGaxCg-6CslS4zvzEtheQOazNlxM,376 +jedi/third_party/typeshed/stdlib/3.7/contextvars.pyi,sha256=buC4sVtTFV1eqWvJ3Xd53ymdER7bHRY7RXGcN21HF68,1514 +jedi/third_party/typeshed/stdlib/3.7/dataclasses.pyi,sha256=KYjW7iSlZ42rrD1Z34CvtIjUSzOTm2lcLozDVSAOkSo,2737 +jedi/third_party/typeshed/stdlib/3.9/graphlib.pyi,sha256=Spt7N2qI_jP87TY6Oqfn5q1JKonbPmQnGn7yMBPAq9s,592 +jedi/third_party/typeshed/stdlib/3.9/zoneinfo/__init__.pyi,sha256=QjdggL46x_feOdxU4aioHhVM0-obo-vi_N-SBsgR-S4,1183 +jedi/third_party/typeshed/stdlib/3/_ast.pyi,sha256=xgO_RWFNlpKCgJ-N8Oz0MLoNxsluUZK6-LgvM7GHk9s,7851 +jedi/third_party/typeshed/stdlib/3/_bootlocale.pyi,sha256=0g5u3OybUgqbQUaK32XpV2OY1vMr_tMyvv-xWePyn60,63 +jedi/third_party/typeshed/stdlib/3/_compat_pickle.pyi,sha256=bgMC5XUuSbLzpZNo1xi_Lmdhuk7YEkNehIU3VbxJF0A,388 +jedi/third_party/typeshed/stdlib/3/_compression.pyi,sha256=2oEmy37B7HOAAXqPvDtdlBgIuI4veE3Zn_ahmMZek9I,761 +jedi/third_party/typeshed/stdlib/3/_decimal.pyi,sha256=gKb8JF9qlgs-k7lWkc6x13adH918J9IBSh2vJJO2-Es,22 +jedi/third_party/typeshed/stdlib/3/_dummy_thread.pyi,sha256=eurqRwuKXCugsXKBB_45dhYwMfGsEOI3JM36MU0_38c,800 +jedi/third_party/typeshed/stdlib/3/_imp.pyi,sha256=ncrUaJ1P-gDktpPYaAEDEd6O0vWZoQbZaEUYkJJEUi0,705 +jedi/third_party/typeshed/stdlib/3/_importlib_modulespec.pyi,sha256=2H1Bqwkmzeu7VdhBr_ZD4YGYLfFCcQPtfrs8Vc8-B0I,1586 +jedi/third_party/typeshed/stdlib/3/_json.pyi,sha256=TLqFcbDts6gPDQGO1UBQuu7Sgr707zI5mIsc0nE4468,1124 +jedi/third_party/typeshed/stdlib/3/_markupbase.pyi,sha256=bxmSN6aw-V3qwShSR82gk2l_ZrCF0KGQhP3baRo7TPE,256 +jedi/third_party/typeshed/stdlib/3/_operator.pyi,sha256=cW8gndiQv2Ge6SxDU1TYCmdDJ1jjCpRO4Blp3IzYBJ4,1310 +jedi/third_party/typeshed/stdlib/3/_osx_support.pyi,sha256=UzxlYXvWxj-409uwv1Qlrt2Dz7SljoBGbxpQn2Rgeok,1796 +jedi/third_party/typeshed/stdlib/3/_posixsubprocess.pyi,sha256=CSDkhdm87FCJzDPSZ7kRSuruLaUJCFO6-grnXJAohPg,557 +jedi/third_party/typeshed/stdlib/3/_pydecimal.pyi,sha256=droa2p0DPjlLpftTkyseRtRiWWwFEFuBmsKmAEBykHI,157 +jedi/third_party/typeshed/stdlib/3/_sitebuiltins.pyi,sha256=joHmJtcwuqTJzYC9Yz8ExsAOrT2ImObL7RzK1J_YKmA,534 +jedi/third_party/typeshed/stdlib/3/_stat.pyi,sha256=rOfqArOAol_zPagL6lyRodbmL93Qsb1vl1UMNz9x5QE,1179 +jedi/third_party/typeshed/stdlib/3/_thread.pyi,sha256=bwRRV0tuEwneVnc4LSBeYjCZp6rbUVAxzFU17xRxVzc,1451 +jedi/third_party/typeshed/stdlib/3/_threading_local.pyi,sha256=nxpI1n5Bmux1ymFk1lGgWgzSzqFHIDvH_y_FJzSGMqc,490 +jedi/third_party/typeshed/stdlib/3/_tkinter.pyi,sha256=yIaAQOXeQdYw5KzaBCpRx8VZJ67khvPs95zlZI9cM4k,2531 +jedi/third_party/typeshed/stdlib/3/_tracemalloc.pyi,sha256=0mp3PJAcjINGNLU8pmyA2_18SfGMgVN8p72QNz-TfPU,563 +jedi/third_party/typeshed/stdlib/3/_winapi.pyi,sha256=e80Wj1rKs41wv4T4tVVGedGngTWfXB8Yal7OS3iXRKs,4507 +jedi/third_party/typeshed/stdlib/3/abc.pyi,sha256=sFDzu6W3-WI8VhWu-3f7oy89xu-T2ad5AkZOr-oaKbk,597 +jedi/third_party/typeshed/stdlib/3/ast.pyi,sha256=rN2zXBxVShE5080Uq3D6J8cFVU0PMumhHOWLGBp6EyE,9090 +jedi/third_party/typeshed/stdlib/3/asyncio/__init__.pyi,sha256=vQqBti5lCesBtJosU6ISR_TaGXUnon5V2niuGmIAFgM,4242 +jedi/third_party/typeshed/stdlib/3/asyncio/base_events.pyi,sha256=z3HWCTbmefm0vunGqkQwL2fvzDwHWxeQGyI6FtzfHIA,14112 +jedi/third_party/typeshed/stdlib/3/asyncio/base_futures.pyi,sha256=S2IqXCOFikl3_yUkCuiONt0mvmvxW_OcK5ZhlWv38cQ,733 +jedi/third_party/typeshed/stdlib/3/asyncio/base_subprocess.pyi,sha256=BY-Dg-c_bEEe9OMq3LkTlsf3b9GfDTy9GpG6ff7os-Y,3239 +jedi/third_party/typeshed/stdlib/3/asyncio/base_tasks.pyi,sha256=WgkoJoXQj11Dp8EFMNpux0aZD5C62kXOyEVuPgTHUNU,412 +jedi/third_party/typeshed/stdlib/3/asyncio/compat.pyi,sha256=5RcYr5xnYP7yXeBN4CKGSeYeNXUNNU1OCkeliI5fmXM,180 +jedi/third_party/typeshed/stdlib/3/asyncio/constants.pyi,sha256=mi5hQw879HlxJe6jIB5ZGcFF7IwUSNwV39dUXP14JG8,327 +jedi/third_party/typeshed/stdlib/3/asyncio/coroutines.pyi,sha256=KSMF7NRW8a61FFkMXSbI-I957V8-cnI4vr-y5jPIcLE,226 +jedi/third_party/typeshed/stdlib/3/asyncio/events.pyi,sha256=GfQt1pVJScwwdoKJOILTBFFA4ew_qTSYteCM-W6laQI,18648 +jedi/third_party/typeshed/stdlib/3/asyncio/exceptions.pyi,sha256=dROhdLRjCGhqH7bI3_yUbPBr7aIKBM1Ks_iWi-7HOq0,562 +jedi/third_party/typeshed/stdlib/3/asyncio/format_helpers.pyi,sha256=3fVMBIqfwuM6eLWjnNv0OFdNZEs9uk2on5NbaeCTW5Q,921 +jedi/third_party/typeshed/stdlib/3/asyncio/futures.pyi,sha256=pKyo4aaJBbPLHDZepPclI8A-2flbzGx60pWiCn49dTw,2581 +jedi/third_party/typeshed/stdlib/3/asyncio/locks.pyi,sha256=UFo1H-sg4cOAyT0rj-Rbm_58FyR73NGvXT70-vbWMW4,2806 +jedi/third_party/typeshed/stdlib/3/asyncio/log.pyi,sha256=Ql97njxNKmNn76c8-vomSAM7P-V14o-17SOIgG47V-U,39 +jedi/third_party/typeshed/stdlib/3/asyncio/proactor_events.pyi,sha256=VcgZHD_Vsu0VSBYYXa4pZTwntgr9okSrLZs-6xcR9o0,2783 +jedi/third_party/typeshed/stdlib/3/asyncio/protocols.pyi,sha256=SBG_ZeDS4RSwq6vj0uppbRezcJKyjRGpqnWOSWSqlqY,1072 +jedi/third_party/typeshed/stdlib/3/asyncio/queues.pyi,sha256=KWOi5Y1t8_v4Nlf0T50xiTi4okN1kxpU1kw74TnBLIM,1166 +jedi/third_party/typeshed/stdlib/3/asyncio/runners.pyi,sha256=8fwyzaEpkiQuUJBAJGdA4QLa6keFBm_ojm0N5KQn2pk,314 +jedi/third_party/typeshed/stdlib/3/asyncio/selector_events.pyi,sha256=ajnESF-_WSF-65nUWRQKJJrq95fJAkO6mYUIdpDWRL0,215 +jedi/third_party/typeshed/stdlib/3/asyncio/sslproto.pyi,sha256=bXJ1AhfKicw_i991U4uAnI6LIC5NCTotbzIX_o_tEAY,5415 +jedi/third_party/typeshed/stdlib/3/asyncio/staggered.pyi,sha256=5ysf7K4OZzJoslO6Jyutra3HVjrgdYXzOxBDc9SpnS8,396 +jedi/third_party/typeshed/stdlib/3/asyncio/streams.pyi,sha256=Vjlv0BjrBw-jAlUE7AYF-V3-GqMYJO8waNOpLBRKs3M,3941 +jedi/third_party/typeshed/stdlib/3/asyncio/subprocess.pyi,sha256=qeMUmUnuAWmms9LYfR5Ldgao3tWDPhxuwprVQ9rRE6I,2265 +jedi/third_party/typeshed/stdlib/3/asyncio/tasks.pyi,sha256=NR-30BS6Tgn5Xd5qj_HMiN1yvd6xT0D21HYLmSkOO8Q,7153 +jedi/third_party/typeshed/stdlib/3/asyncio/threads.pyi,sha256=xNqf8z1pbCIoeaO4w4YTzcuqOoGKbb1bJ7KjXpP7vWU,194 +jedi/third_party/typeshed/stdlib/3/asyncio/transports.pyi,sha256=81VjVySq54ht5UnyaxEaYhjBgt263z0nUGuwl2zNWvc,1886 +jedi/third_party/typeshed/stdlib/3/asyncio/trsock.pyi,sha256=Gywt7hJSyhtc9oIGAhTZUls1u_5C_AtmGKlOCkdqyIk,4582 +jedi/third_party/typeshed/stdlib/3/asyncio/unix_events.pyi,sha256=xnaVSEGzJn3AKT0eCG9g0xacrj5EG9K6fA8MNbD2EEw,2144 +jedi/third_party/typeshed/stdlib/3/asyncio/windows_events.pyi,sha256=SQQ4tw0uGxFkIaFjMhVwqHlpU2kyBt-7Zb1qUKbNmGM,3554 +jedi/third_party/typeshed/stdlib/3/asyncio/windows_utils.pyi,sha256=sPyeKsJrsOsSHFop6mMTdPdyiOasrfvAVWRCLVDDruY,983 +jedi/third_party/typeshed/stdlib/3/atexit.pyi,sha256=NLT6GZV7DmuNcHb-lTn2R5mT-V12bibRWe12ittWOeo,271 +jedi/third_party/typeshed/stdlib/3/builtins.pyi,sha256=OnHDGVqjdghwpPVhsTK7DHElTolNSs8MvfAHEwoLw8o,54524 +jedi/third_party/typeshed/stdlib/3/collections/__init__.pyi,sha256=CRVuUl5jM4mQlf1i1Xsn2zm1KljSS0XzEvNpAI2kPRk,14502 +jedi/third_party/typeshed/stdlib/3/collections/abc.pyi,sha256=dVx25fVnQBL30UcThhhbW3wcpH5Ru5EM_hW448QeFPg,744 +jedi/third_party/typeshed/stdlib/3/compileall.pyi,sha256=YYPeCHqW1sy_ecwW2gdNA4JErmOMqzizTus5hYh__IM,3367 +jedi/third_party/typeshed/stdlib/3/concurrent/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/concurrent/futures/__init__.pyi,sha256=ZyXXp9i057jbhjo8WeR9SYBT0dhFnwyZ-L1cJP4pQWg,629 +jedi/third_party/typeshed/stdlib/3/concurrent/futures/_base.pyi,sha256=4tKni6V8xudjGjgdqmUCyWa4aQFINZrLlzIzJegI4O4,4676 +jedi/third_party/typeshed/stdlib/3/concurrent/futures/process.pyi,sha256=ZJi1cVZxQPFoJnHUunWf_kUSlEVX_NBbj3asiI8_shM,804 +jedi/third_party/typeshed/stdlib/3/concurrent/futures/thread.pyi,sha256=7thIcnDTCR-0-lB5jJFN4xWzpt4MoQiGy2vKkymPA4w,1421 +jedi/third_party/typeshed/stdlib/3/configparser.pyi,sha256=v_JccqizVrtoccBdWqJSDGiZUbDQaXu2Eoh96AI3EHM,10095 +jedi/third_party/typeshed/stdlib/3/copyreg.pyi,sha256=-GmSIudewg4xblZXLSvErhpunAMi1bsqdQUEXujpFlI,739 +jedi/third_party/typeshed/stdlib/3/dbm/__init__.pyi,sha256=fxHKjeckXzF2s17gpHZ7mD2CKOw42qFaX8oTsa2wUGM,945 +jedi/third_party/typeshed/stdlib/3/dbm/dumb.pyi,sha256=f1YcbTYCzE8bcoC-J9jL8-tKlwTMMV0JBjGFzuj4EQk,1010 +jedi/third_party/typeshed/stdlib/3/dbm/gnu.pyi,sha256=zMcocoAPlm0vkOt5kqWVu6eBe7HZ74A3xl0W7PgrpSc,1363 +jedi/third_party/typeshed/stdlib/3/dbm/ndbm.pyi,sha256=mCED-czlQBkm28csVVuyWFXNQ282xg1sUAnsCOvaMP4,1236 +jedi/third_party/typeshed/stdlib/3/distutils/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/archive_util.pyi,sha256=WaUu32XUCstlA4zE0WS-PvA9UF0mdyuDgDbZZd1LN0A,447 +jedi/third_party/typeshed/stdlib/3/distutils/bcppcompiler.pyi,sha256=fge2cMbG4jp--o0I2zNcwykh24tJWZtk6leQgAH2NJw,78 +jedi/third_party/typeshed/stdlib/3/distutils/ccompiler.pyi,sha256=N38wYG41RyiEuowx_ZvpZIqVVZYiz1NGcGHHDJ9MbWs,6449 +jedi/third_party/typeshed/stdlib/3/distutils/cmd.pyi,sha256=wquIAgFbqLNsoHDPGQm3WuGU13054o1u4FGHprpI_9Q,2803 +jedi/third_party/typeshed/stdlib/3/distutils/command/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/bdist.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/bdist_dumb.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/bdist_msi.pyi,sha256=sDSqH7TRcOiXC5S4VXxJ_YHB-WFPpa1fo8F8g5XeV3Y,182 +jedi/third_party/typeshed/stdlib/3/distutils/command/bdist_packager.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/bdist_rpm.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/bdist_wininst.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/build.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/build_clib.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/build_ext.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/build_py.pyi,sha256=lBXy2QTRaedFpVnXmdyJonKRrnEjAKYViarVHccdb1w,217 +jedi/third_party/typeshed/stdlib/3/distutils/command/build_scripts.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/check.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/clean.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/config.pyi,sha256=DV1oMIztVDdk46W43HGioP_n6b3x9FgSqvFr2rwPVoY,3059 +jedi/third_party/typeshed/stdlib/3/distutils/command/install.pyi,sha256=z_gxnwFlHhSicrx9YRtsJnJQ1YG7mu6zERZwZ-1RJi8,365 +jedi/third_party/typeshed/stdlib/3/distutils/command/install_data.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/install_egg_info.pyi,sha256=WcnLycNSSWSZ8Z_vHIohu0-3qKnCih2qoJaGBPtjQGY,380 +jedi/third_party/typeshed/stdlib/3/distutils/command/install_headers.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/install_lib.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/install_scripts.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/register.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/sdist.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/distutils/command/upload.pyi,sha256=NQ4QYCmhC98gBW2Fv2LW0xzFNZI-9gI6AjTW_UqeOJY,279 +jedi/third_party/typeshed/stdlib/3/distutils/config.pyi,sha256=M4b9_7PKsO74DhsIBFoTAB5I99hXrlBJcjxghWhI8vc,523 +jedi/third_party/typeshed/stdlib/3/distutils/core.pyi,sha256=lq1S8lRqeGqK6U3z7_UsVg2TR7xbx7Z2YXJ0ekIo9Vk,1688 +jedi/third_party/typeshed/stdlib/3/distutils/cygwinccompiler.pyi,sha256=Y7qhVOqXrkPT0yyQnudNCtTzBYC2lzS38HB5Mh45zEI,138 +jedi/third_party/typeshed/stdlib/3/distutils/debug.pyi,sha256=7_zuriUBqHbc62x7tCONq7LQXLuK_hCaBK0laoR3HeY,12 +jedi/third_party/typeshed/stdlib/3/distutils/dep_util.pyi,sha256=QCheHEDF7waISF42_aaumqvVOIcTw-yh5e5-CbPvQ2o,252 +jedi/third_party/typeshed/stdlib/3/distutils/dir_util.pyi,sha256=0nHuLCqZ36gvDVaF6PoC76_JOC3v6P_310eFauW1ZDM,555 +jedi/third_party/typeshed/stdlib/3/distutils/dist.pyi,sha256=a5bZv3WJK1cAA-rXLz2WRgvWRr_KrNShz1ho37Z73nA,2557 +jedi/third_party/typeshed/stdlib/3/distutils/errors.pyi,sha256=l1W_FgoP9L-D-hEPFA2BzZuybjN0lV4WBXl0VJ-k7J8,852 +jedi/third_party/typeshed/stdlib/3/distutils/extension.pyi,sha256=DSCTrWp6UVQGcvlVM-07HCJUNMT7ggxg8-gGBL1hKlg,736 +jedi/third_party/typeshed/stdlib/3/distutils/fancy_getopt.pyi,sha256=ai-0E83HnTOcYCWO9zq4tDVvnef04SRRoa-F8baso_o,859 +jedi/third_party/typeshed/stdlib/3/distutils/file_util.pyi,sha256=RFpiizXMdhhcji98pIscfHW4r6i9KLzM2D15gA6_eow,439 +jedi/third_party/typeshed/stdlib/3/distutils/filelist.pyi,sha256=-WeYFFKsEUUjPvbzeZbVCOKkPV-oqc3RoZvN2SB1VOE,20 +jedi/third_party/typeshed/stdlib/3/distutils/log.pyi,sha256=CXViHmBVIadN4DWrSddT9lN1O1WaNjpxvaz-KUCD8C0,845 +jedi/third_party/typeshed/stdlib/3/distutils/msvccompiler.pyi,sha256=qQLr26msfhjz-omJutWcRHik3shLh1CIt7CDI3jBd3I,78 +jedi/third_party/typeshed/stdlib/3/distutils/spawn.pyi,sha256=iDdi2WvST9yeFDlKWy0Wlye-x67W-ah5nta7EuRW2W4,227 +jedi/third_party/typeshed/stdlib/3/distutils/sysconfig.pyi,sha256=FSdBoSTsVvKAF5D2lkWBwxH15ockfFZv6L06mrgeAb0,620 +jedi/third_party/typeshed/stdlib/3/distutils/text_file.pyi,sha256=vCQLwggDaocAqqR7v1WJjOeS_wgxqjI5xDkyxHJlzcw,716 +jedi/third_party/typeshed/stdlib/3/distutils/unixccompiler.pyi,sha256=R3VKldSfFPIPPIhygeq0KEphtTp0gxUzLoOHd0QoWW8,79 +jedi/third_party/typeshed/stdlib/3/distutils/util.pyi,sha256=KSpX8rQ6qJXqToJBKdwhlpe-jd1QQcacg7wsH_6dKXo,829 +jedi/third_party/typeshed/stdlib/3/distutils/version.pyi,sha256=-MSresMMQbCxM9NDl_Hof4kDdGYmlJmqL8QWHCbliqM,1429 +jedi/third_party/typeshed/stdlib/3/email/__init__.pyi,sha256=5hwilWCw-1AiwWEMUfgVza8eTnH1vsqSL4Dkjx44w4c,757 +jedi/third_party/typeshed/stdlib/3/email/charset.pyi,sha256=tKEN5fdqEvA9KmtWRqGdv7MFSdI8IKcRMrkjzvdUpyQ,1062 +jedi/third_party/typeshed/stdlib/3/email/contentmanager.pyi,sha256=1onMqjP_pwFFABKkXAJMpLssp2eCajYb_DmfdGW88dg,489 +jedi/third_party/typeshed/stdlib/3/email/encoders.pyi,sha256=WIE0oEGqiDZZCnaaUganOj3RIHvNpdH1H6_uYsb9BCU,214 +jedi/third_party/typeshed/stdlib/3/email/errors.pyi,sha256=5uMjZNDOPcQBIOB08smMOo19sHIAc_Zcbsbz4OAjrlM,833 +jedi/third_party/typeshed/stdlib/3/email/feedparser.pyi,sha256=-El1uWq32_69H0WtIpGT6eiEsyCQRcHuGUxKSlE7OdI,823 +jedi/third_party/typeshed/stdlib/3/email/generator.pyi,sha256=aBWIi9nhJSWROoxD0pIk0TMLt0j2epr_ASen8iiUKPs,967 +jedi/third_party/typeshed/stdlib/3/email/header.pyi,sha256=N32hw4GGgE82e7R8dpDHQnQXLf6YABLJ5OGWFuvwQNw,1025 +jedi/third_party/typeshed/stdlib/3/email/headerregistry.pyi,sha256=M9ZBxQ_YJIe_lWFLyr0AkvgkIy0ch1rhnZm7JNIT2LE,2828 +jedi/third_party/typeshed/stdlib/3/email/iterators.pyi,sha256=kVhrZU7h-D41NBZrgFU_h2xeBs02vSRL56lfxXiRy8g,266 +jedi/third_party/typeshed/stdlib/3/email/message.pyi,sha256=4OVN83CSaSRycbt4NmwrHwB643-65ibkluIiQqNa4Os,4681 +jedi/third_party/typeshed/stdlib/3/email/mime/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/email/mime/application.pyi,sha256=1nvquRMV3pfipzE_Db77xQA2YiVshkslbrR6KJZXN2k,499 +jedi/third_party/typeshed/stdlib/3/email/mime/audio.pyi,sha256=hsj22zxJwLlF6TPXDzRMksu4oUoxnypUshYenxXSkE8,502 +jedi/third_party/typeshed/stdlib/3/email/mime/base.pyi,sha256=PLTWaSsib758s-xoTPQE43teS-vStf7nOoPLiq_evHg,325 +jedi/third_party/typeshed/stdlib/3/email/mime/image.pyi,sha256=DpDOxmW7k3nF8WzPhaeLLyYWNeYIQgW1v5OU7v3U4_E,502 +jedi/third_party/typeshed/stdlib/3/email/mime/message.pyi,sha256=BbZ7ULwq3vT6-MPS2QFmkIuWqxgC9vDLWD9x0zv9oXk,292 +jedi/third_party/typeshed/stdlib/3/email/mime/multipart.pyi,sha256=CTHQJbuC-ZAgPKymxr62DjsslnF4P45RZQ_a17481UA,507 +jedi/third_party/typeshed/stdlib/3/email/mime/nonmultipart.pyi,sha256=gAN-hOmjxz3lcG8FK3lbvlEOoz_yhG9rKIslq3WLil4,76 +jedi/third_party/typeshed/stdlib/3/email/mime/text.pyi,sha256=uqvOSr2SoHrbTcLikt1P7Gm0L9sjyy6c5TeuX1cbvBY,297 +jedi/third_party/typeshed/stdlib/3/email/parser.pyi,sha256=p-CbSumd_5JYzgkS81GOZofpQuameh6ajoLtJifdpyQ,1328 +jedi/third_party/typeshed/stdlib/3/email/policy.pyi,sha256=sFF7jOYVcLw_gIS27K7CxMFbz62BbWXM_qVPQQakyx8,2159 +jedi/third_party/typeshed/stdlib/3/email/utils.pyi,sha256=V-8KVirBmcrTekDW6TW7gqkV3bPCh2l9RJKtGHCyg5M,1832 +jedi/third_party/typeshed/stdlib/3/encodings/__init__.pyi,sha256=q95Eqw-U2YICiPawMyarXy1iWCXsz1wV_793ZSg_4ek,184 +jedi/third_party/typeshed/stdlib/3/encodings/utf_8.pyi,sha256=QvCf0jkCM3g7fA1rzv7IB8onY_ciFLPaySvkBQ95aOM,561 +jedi/third_party/typeshed/stdlib/3/enum.pyi,sha256=iY_vEF2WHiMhmgZTtBGnE_XdzdPx90709cxW0BP9o4M,2643 +jedi/third_party/typeshed/stdlib/3/faulthandler.pyi,sha256=jn6gMdF0GEZljfFTc1splgef8zIo99X1H44qgWxU8sE,644 +jedi/third_party/typeshed/stdlib/3/fcntl.pyi,sha256=YNY4OFCzwUNVzqaocJgNzrS7kwsRrKCzFmXfv9ukHUc,2244 +jedi/third_party/typeshed/stdlib/3/fnmatch.pyi,sha256=_sH2M1wxuUFryRYDO_K2eXtxxJPSRHHRRuz2nzUqf6Y,257 +jedi/third_party/typeshed/stdlib/3/functools.pyi,sha256=twLVNCB5Heu0Y1yfJbaDdFbDCP0kX9PYYg4vumV3iKI,4613 +jedi/third_party/typeshed/stdlib/3/gc.pyi,sha256=mfh-9jtsLPOaFLuBwkRgt1rPjpuRf6GelyMb8PJjokc,1135 +jedi/third_party/typeshed/stdlib/3/getopt.pyi,sha256=QUWgEAjkZEgYEDX83HNFzlLqoM7tWZfu35croNXpYUk,352 +jedi/third_party/typeshed/stdlib/3/getpass.pyi,sha256=VZjU7IEeEVtL8nHUcWRbbOkNYAegfSRiTDc5QFY-BuM,178 +jedi/third_party/typeshed/stdlib/3/gettext.pyi,sha256=hI05wMFAUpQqs9xphkmJdM2UbLKcvQ9--PNPwOaPQiQ,3208 +jedi/third_party/typeshed/stdlib/3/glob.pyi,sha256=l-vPtsxqdYR6852YtsLh8m9dS4g2OvnU2hDUPHnwRw4,442 +jedi/third_party/typeshed/stdlib/3/gzip.pyi,sha256=hxBMzBPUEElN-Q0CyO7__tQ8cOH0kboT5Oqqah3HLms,2909 +jedi/third_party/typeshed/stdlib/3/hashlib.pyi,sha256=a1LtNJGGukXpq3P09xQvoAfU9aDsTR3nZ09T7EyFOL8,4195 +jedi/third_party/typeshed/stdlib/3/heapq.pyi,sha256=ZXIM49YLZnLiEMSQlWt9uUzX6fxgNXH5HIQwBVhcneI,798 +jedi/third_party/typeshed/stdlib/3/html/__init__.pyi,sha256=qKsbjN2OJn_AoUr9a4zdFC8juUG9ii6rqYp3EQTSG-Q,122 +jedi/third_party/typeshed/stdlib/3/html/entities.pyi,sha256=ap6jI6Fw355WC0vSd-WGc957p_KpAJDGONUeAlItYzI,136 +jedi/third_party/typeshed/stdlib/3/html/parser.pyi,sha256=odnTynZ-Cz1KlNCzxR_nl3NY_vEHlg1iha4DGTHCX0Q,984 +jedi/third_party/typeshed/stdlib/3/http/__init__.pyi,sha256=bE3i99Lkeat-Seo0TTAm-dDauYTTNtih512Tt6ij9G8,1940 +jedi/third_party/typeshed/stdlib/3/http/client.pyi,sha256=N-CIvhqvj7MhnMDK-rhyuH5FgsBQCjsTcuP230LsH7M,6034 +jedi/third_party/typeshed/stdlib/3/http/cookiejar.pyi,sha256=vIs7US67GiRUDVmq_qdQ9a48s4Iv73h4_S6D8OYrNcA,4991 +jedi/third_party/typeshed/stdlib/3/http/cookies.pyi,sha256=pfXrzKusaD-sxZsv92P84Vw0sgoGrE-KKhiuXm15vz0,1364 +jedi/third_party/typeshed/stdlib/3/http/server.pyi,sha256=C9LgOZw5ZoAYzbXN-y9BXdS31iaEiwwo-QpynkLS_QM,3036 +jedi/third_party/typeshed/stdlib/3/imp.pyi,sha256=_zd93_282PmC8cqim3FPUsFeYrIl8-dp_WFab_v7nac,2343 +jedi/third_party/typeshed/stdlib/3/importlib/__init__.pyi,sha256=xuq-05qRDyIRBetquMX_JB1dMc_BLme0WAZ3MpggjPQ,571 +jedi/third_party/typeshed/stdlib/3/importlib/abc.pyi,sha256=polXx3wt7y2EQYqfND7eD1OA0hMt1ucG_vQoACtvq4U,3960 +jedi/third_party/typeshed/stdlib/3/importlib/machinery.pyi,sha256=FXq0GgiGYrJRtZKN76mlRVRuum88bdiq_k_ZuF08I18,4006 +jedi/third_party/typeshed/stdlib/3/importlib/metadata.pyi,sha256=e-uCY4Q7Su-dTeeUl-sGUF9F7-70Vd2hf_CQ2Y0IT9k,3786 +jedi/third_party/typeshed/stdlib/3/importlib/resources.pyi,sha256=zYjYz5gFHS1eT5roJ2glvvqhVwjJBkLdxkMYUj3FhJo,1026 +jedi/third_party/typeshed/stdlib/3/importlib/util.pyi,sha256=bqqXx43SyHCCERNe7jA-i6MjfUdyPpQtimtuqqTKF2M,1782 +jedi/third_party/typeshed/stdlib/3/inspect.pyi,sha256=yYThFqYLPUJToz4Fz9LYxI-ZJ89k94wqpvhUoK1K-QA,9929 +jedi/third_party/typeshed/stdlib/3/io.pyi,sha256=OQHQh77AvF1wqIqeqUPiHDO9nKgRX45O6ZgOrQap7xE,7499 +jedi/third_party/typeshed/stdlib/3/ipaddress.pyi,sha256=74Fj1yj7-tgOJeSSR8LKvfqIUyY1UzTQN6q_lrUcgpU,5252 +jedi/third_party/typeshed/stdlib/3/itertools.pyi,sha256=Up21RC9zUGPLkg3vHuuQ7iSWpphBCeWGBH0Txn1mUk4,4524 +jedi/third_party/typeshed/stdlib/3/json/__init__.pyi,sha256=dWEcInSJdvZapNjd2VvnksUHKyj4_AVSVJasp7uqFBw,1919 +jedi/third_party/typeshed/stdlib/3/json/decoder.pyi,sha256=85tOPFvXraUsmSgW1Sn7FPqKAxSWwPZwjAZ7Aa6CMFM,1090 +jedi/third_party/typeshed/stdlib/3/json/encoder.pyi,sha256=FIZVowFsNCGiAAKqnohbOQkmFKsvujvwkzk2qu0OCSg,779 +jedi/third_party/typeshed/stdlib/3/json/tool.pyi,sha256=d4f22QGwpb1ZtDk-1Sn72ftvo4incC5E2JAikmjzfJI,24 +jedi/third_party/typeshed/stdlib/3/lzma.pyi,sha256=t8jPcnq6o5L1t3nKdw_dmeFFovwCAkQZnKnMltEypTc,4580 +jedi/third_party/typeshed/stdlib/3/macurl2path.pyi,sha256=IjGXxi7XlrbJwIcpfitwuKQWR635yLOf6Sh3N1o25Do,225 +jedi/third_party/typeshed/stdlib/3/multiprocessing/__init__.pyi,sha256=J3pDzfv93b1AS4hf69jhyonvhWmrj1eYbCcnApCWlUI,3802 +jedi/third_party/typeshed/stdlib/3/multiprocessing/connection.pyi,sha256=ReCch6MGZ-m4TSo6bSFHUv9byxu79g2GP7chnYmhD8w,2602 +jedi/third_party/typeshed/stdlib/3/multiprocessing/context.pyi,sha256=LQireotdnvV1vrojP-BTgtSyAZpius2NGrbl1kbUpmc,6698 +jedi/third_party/typeshed/stdlib/3/multiprocessing/dummy/__init__.pyi,sha256=0n5irDRx6L6Q8NlRCxq9w-Dlk4DhJhiGxbFiFTPZSgM,1572 +jedi/third_party/typeshed/stdlib/3/multiprocessing/dummy/connection.pyi,sha256=1h-QyO57HTVQN_OiJjuiNp9QSEkuXKZDdR-jrhasw44,1431 +jedi/third_party/typeshed/stdlib/3/multiprocessing/managers.pyi,sha256=_owX_1Pum7mnYDXFKtk72T8KHfckdNl_hg8FT5AUsoQ,4584 +jedi/third_party/typeshed/stdlib/3/multiprocessing/pool.pyi,sha256=Zz1Lx-lpsQrJHgsAS6xqCUqsh2F-E6lP0Iu_Vc4RXA4,3224 +jedi/third_party/typeshed/stdlib/3/multiprocessing/process.pyi,sha256=Eupk7EcZSXHEe_LcXKt_uY_oYt5ZGpwFoIVVVAiCq78,1143 +jedi/third_party/typeshed/stdlib/3/multiprocessing/queues.pyi,sha256=fXQ9gQc8wqYmEgwBCK6bpaTLUR-tPGAZxtADAw6OdUw,1288 +jedi/third_party/typeshed/stdlib/3/multiprocessing/shared_memory.pyi,sha256=oZEZSEATdR5PqJEtyd6E3ZWsSsM_mjvOW_CRhVjP1JM,1302 +jedi/third_party/typeshed/stdlib/3/multiprocessing/sharedctypes.pyi,sha256=qg970qF7Qy0nnIWPnqV3q0Sycxx0dZAV60Fa88oUyOM,1526 +jedi/third_party/typeshed/stdlib/3/multiprocessing/spawn.pyi,sha256=cMRcm2n3hnOszIByMFkG3H7PatT472zWi-g-s8ZdkPk,690 +jedi/third_party/typeshed/stdlib/3/multiprocessing/synchronize.pyi,sha256=ZUPVL36s3ANf4divriLWpgHYFBva09_09IK7hORTSns,1799 +jedi/third_party/typeshed/stdlib/3/nntplib.pyi,sha256=KSCji19ptgFnr26zKlzgLulK7e0LSfPLZHLBSMKYN54,4325 +jedi/third_party/typeshed/stdlib/3/ntpath.pyi,sha256=MZG5ucT7dEvAJYFsJHN7SHM2Frs4Nyf3eLm6VqhUAX0,4721 +jedi/third_party/typeshed/stdlib/3/nturl2path.pyi,sha256=E4_g6cF1KbaY3WxuH-K0-fdoY_Awea4D2Q0hQCFf3pQ,76 +jedi/third_party/typeshed/stdlib/3/os/__init__.pyi,sha256=sqb9_zZjeE44gdDnephPQIj8rzLZU8WIuVAj13qcdu4,28846 +jedi/third_party/typeshed/stdlib/3/os/path.pyi,sha256=MZG5ucT7dEvAJYFsJHN7SHM2Frs4Nyf3eLm6VqhUAX0,4721 +jedi/third_party/typeshed/stdlib/3/pathlib.pyi,sha256=WxnASz0QaegMLAzqln-6G529DGVub-Jg8D8lyWGjGSM,6899 +jedi/third_party/typeshed/stdlib/3/pipes.pyi,sha256=cgke6nDBXMhLkpcUUsV_y61rc2ujKCydFa6kl3oYVdU,518 +jedi/third_party/typeshed/stdlib/3/platform.pyi,sha256=_a4cugd9uoy8pzW0KE0U0feB2O5k4Co1X_VeKHbgbGQ,2272 +jedi/third_party/typeshed/stdlib/3/posix.pyi,sha256=YNT-Sw-8HFLKnXeCg2GRN97r0K6piolWHPuiD-qPK6k,2810 +jedi/third_party/typeshed/stdlib/3/posixpath.pyi,sha256=MZG5ucT7dEvAJYFsJHN7SHM2Frs4Nyf3eLm6VqhUAX0,4721 +jedi/third_party/typeshed/stdlib/3/queue.pyi,sha256=W1aF5vLpBMTnl4ObekHw4B67WbhjoUSukrz3g6opa0k,1884 +jedi/third_party/typeshed/stdlib/3/random.pyi,sha256=G3NB5JMkfENo8v1u6X00c1-V2HEX4W9fRqcNibNdGNc,3901 +jedi/third_party/typeshed/stdlib/3/re.pyi,sha256=E1wPk3W7Mp-0VSatzEl1WR058yXWhyau-vJfFsr4Ovo,4442 +jedi/third_party/typeshed/stdlib/3/reprlib.pyi,sha256=Hn5K7nvw9i4uwyrxH7BkSSbA8_na_24t6YJ8YleX0cQ,1228 +jedi/third_party/typeshed/stdlib/3/resource.pyi,sha256=Je96IzWSE5BZkL25TXCyg3wVbGO2zMHGrLjjqKgl-gY,1243 +jedi/third_party/typeshed/stdlib/3/runpy.pyi,sha256=kYMEMd1kHr9LmvQzt9-s4xXrAAth_upzRtxbkXtl_r8,746 +jedi/third_party/typeshed/stdlib/3/secrets.pyi,sha256=38xdvca6iVqHllAA8pfiRXMnSFEZ_AakTCHjVjksHIo,467 +jedi/third_party/typeshed/stdlib/3/selectors.pyi,sha256=5Mg36Zke6M8UL4t91QuDDY5LX6kHwgluUwr4zGSezBk,3644 +jedi/third_party/typeshed/stdlib/3/shelve.pyi,sha256=9NdL2QZYd9Wtqsz991GfroFbZ553lesYsKujKZCrPbI,1605 +jedi/third_party/typeshed/stdlib/3/shlex.pyi,sha256=pVtr3Hc4_Wx7qgcfQPf6DepYemtlrQaZ58l7jckycVI,1325 +jedi/third_party/typeshed/stdlib/3/signal.pyi,sha256=aFVl8Op3n4JAQhj_jEta45aGGmXQciIRswTbmx5VJ68,5209 +jedi/third_party/typeshed/stdlib/3/smtplib.pyi,sha256=DobAN5UFsK41cOSl_hPqRBQskhdlTxUPTbJBH3iH7-k,5606 +jedi/third_party/typeshed/stdlib/3/socketserver.pyi,sha256=7ASKFWF67wQOePi5HR3RqEuAAb2xqkf1YrRhlmNS4RQ,5367 +jedi/third_party/typeshed/stdlib/3/spwd.pyi,sha256=aAmkS56-90q7hfKMkpAmUifbEQy32e2c-gW_KVHTEn8,310 +jedi/third_party/typeshed/stdlib/3/sre_constants.pyi,sha256=j5xp3zUP6iox81-rrMQSi6OqC7s99PXncyL6YCjwMAc,3348 +jedi/third_party/typeshed/stdlib/3/sre_parse.pyi,sha256=JYDEHNQG-Tw_Bwj77fpG7I-7bIk2skV2eMjIQ7ggbcI,3820 +jedi/third_party/typeshed/stdlib/3/stat.pyi,sha256=WYG0DONENxta5efsANvcWTQiy_kkB5NjSYl7oJBgrv8,1805 +jedi/third_party/typeshed/stdlib/3/statistics.pyi,sha256=nVUX7Tcg6mZeuTzdOWCDLH-jULHlCCOf-s0RnPhK0K4,3149 +jedi/third_party/typeshed/stdlib/3/string.pyi,sha256=CNCZS22ux7JIYv0mm9KGJ1sPWk4-ziwN13SKAC0u36M,1423 +jedi/third_party/typeshed/stdlib/3/subprocess.pyi,sha256=aVTeOMtFq4uJMnSXkZG9YA8jjXdhxxY_yjo4e9mLMqI,34320 +jedi/third_party/typeshed/stdlib/3/symbol.pyi,sha256=6iOqBcaia5f9UZcXxZaDy9qBA4Boe35Uy4WxQmhFWcg,1383 +jedi/third_party/typeshed/stdlib/3/sys.pyi,sha256=7WyG6k19z77CBdUUkNVQiAR_YxpKCamnsxjsFX_06MU,6337 +jedi/third_party/typeshed/stdlib/3/tempfile.pyi,sha256=FyzDCJDCZvkXsxr_0arm8Q3vMKgWO58vw4LPl8ZghT0,11371 +jedi/third_party/typeshed/stdlib/3/textwrap.pyi,sha256=SA66hWuD6KU0BfK8EUpEwnm8tKgyWbYcHJVaj_qXdzM,3234 +jedi/third_party/typeshed/stdlib/3/tkinter/__init__.pyi,sha256=zXaLTPom1qPFajA0dc2Ic0_43bCd0gQqvUsu1NhtsvE,110919 +jedi/third_party/typeshed/stdlib/3/tkinter/commondialog.pyi,sha256=GXUxL-Y2G2Y3DgfMNMNpCUzuRnZ746Iij-R56ZXCu6E,277 +jedi/third_party/typeshed/stdlib/3/tkinter/constants.pyi,sha256=WfBSeKTB52NfPVFdsdaRrnsmDOvT3FoP8j4QK1T3yxU,1886 +jedi/third_party/typeshed/stdlib/3/tkinter/dialog.pyi,sha256=SqRt2lN2RMOtj5B9Q1h73xcxbVJTfZ0bYzfgH0nnNpY,291 +jedi/third_party/typeshed/stdlib/3/tkinter/filedialog.pyi,sha256=xM4DePZxB81ctUBT5Wp6hxfQAQAXqunHfv5idcVx71g,2247 +jedi/third_party/typeshed/stdlib/3/tkinter/font.pyi,sha256=HVeakp_l7Vqh6sbakpZ6XlilT2oIQhQA5AajfLhANtY,3812 +jedi/third_party/typeshed/stdlib/3/tkinter/messagebox.pyi,sha256=Q0-rK5wXBbbzvx4Mpf92dITPgL7Lwjw8t4CLfpYV0hw,1150 +jedi/third_party/typeshed/stdlib/3/tkinter/ttk.pyi,sha256=f5W5gEAauDlNzBBJ7e43tuyu9_SzdA4rWxRhhOHlGvE,43748 +jedi/third_party/typeshed/stdlib/3/tokenize.pyi,sha256=MKMkZ5E-aZIObWJom5U1-4JcKMEUuykkKDNs1tcko2M,3110 +jedi/third_party/typeshed/stdlib/3/tracemalloc.pyi,sha256=VqPd3kiHw2LM5yb0YDtj_Rx-kSlOUon2W3cSu1d49KY,2795 +jedi/third_party/typeshed/stdlib/3/types.pyi,sha256=KsTXLa4cimtNz6HGMOn6whGh07tR0NJkc1_PabY6l3A,11027 +jedi/third_party/typeshed/stdlib/3/typing.pyi,sha256=lh5-KS3cFrYby3k1Etdksa6_IZBaTd0V-8o3aV_ztcc,25040 +jedi/third_party/typeshed/stdlib/3/unittest/__init__.pyi,sha256=mot-stW02w3PxX7Q8XcXM-wNPkyICkUEcthaTuzIMh4,387 +jedi/third_party/typeshed/stdlib/3/unittest/async_case.pyi,sha256=8FL7FHo9f5ZAO1veWlhC6r1WXpC4JBSZ9jAGr5VBrhs,372 +jedi/third_party/typeshed/stdlib/3/unittest/case.pyi,sha256=h5wnJUGhA_Jw3GstvFcaWX5jXfFnXnZcODW3nLzfP7o,12483 +jedi/third_party/typeshed/stdlib/3/unittest/loader.pyi,sha256=8wcy7fsfMDuHjVSNzD6bwmPmnUGiNG0PTLz0HpLjDWU,2067 +jedi/third_party/typeshed/stdlib/3/unittest/main.pyi,sha256=TM86Ac-1qF3P9JogJ2rYwE3E4dKi7ke1G8RwLeBEiaA,1691 +jedi/third_party/typeshed/stdlib/3/unittest/mock.pyi,sha256=dXwIB0Naj-0TjlamsKQcrlNRThPHW3nvBHtUt28nR6A,14809 +jedi/third_party/typeshed/stdlib/3/unittest/result.pyi,sha256=v11JXslVniIzkCYnUnalSJ2ndJc4I-Bxt8_Mie_6teI,1859 +jedi/third_party/typeshed/stdlib/3/unittest/runner.pyi,sha256=l-eVrz13ec-okAuc0MGdYbo192H2ep3adVycY6dnUYE,1339 +jedi/third_party/typeshed/stdlib/3/unittest/signals.pyi,sha256=uL7z9v81sng6PtI-FqweH7jef7o41fQD3AL8WoIxuFU,402 +jedi/third_party/typeshed/stdlib/3/unittest/suite.pyi,sha256=YPg_NeVuWOBeQCdI_NHPFyRDhsLy6E0rgmH96jNyj58,892 +jedi/third_party/typeshed/stdlib/3/unittest/util.pyi,sha256=rs8aUKwYgAQfqWYmsy-OEMdcNgeBOMktmf-wmFjd67c,906 +jedi/third_party/typeshed/stdlib/3/urllib/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/urllib/error.pyi,sha256=9VToSt5hDZo_aKbL2MRVwQ019iJ9LFmlaJg5hPOxTsA,391 +jedi/third_party/typeshed/stdlib/3/urllib/parse.pyi,sha256=yj_xX2qv1FoEtJJOk7naDIRVxuGKKwuTWolsztAekG4,5436 +jedi/third_party/typeshed/stdlib/3/urllib/request.pyi,sha256=-tFU7MccfufD8Iss9KyLS0uoWYmX_q3lKeqZJf_x6AI,15581 +jedi/third_party/typeshed/stdlib/3/urllib/response.pyi,sha256=lFv0RWV2D9QrkSrXlCD_Mx9iKdCz6TkOuTy6BA4bdcE,2107 +jedi/third_party/typeshed/stdlib/3/urllib/robotparser.pyi,sha256=vTWHNtcRWcgDk3DwlPv2ULB913AI8l6JpbdMAaTnlJc,704 +jedi/third_party/typeshed/stdlib/3/venv/__init__.pyi,sha256=f_2SVkhePBtzrvKVcVVyS79VKHy6V37E_99ToJRmnXk,2967 +jedi/third_party/typeshed/stdlib/3/winreg.pyi,sha256=Yz2wG-34BwXWx4JaU2TqPjHlLm_eiPoALQm97KGcI2w,3779 +jedi/third_party/typeshed/stdlib/3/xmlrpc/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/stdlib/3/xmlrpc/client.pyi,sha256=iak5sIIqou0OLODL8DPFbpAAFO3_k8SmIHmgLvPYcbE,12285 +jedi/third_party/typeshed/stdlib/3/xmlrpc/server.pyi,sha256=OguOW818SOafYK9lArrfOt7Nw7NgfaGcx9ERt1rBxWo,6810 +jedi/third_party/typeshed/stdlib/3/xxlimited.pyi,sha256=IeL0poSgoE8vvdo93lscYL9hdc2xqfSgnzzb1iQ6hgc,211 +jedi/third_party/typeshed/stdlib/3/zipapp.pyi,sha256=gjXsMJc9OrQhFpRaUVZrrN7ACwo-oMr54g0KTKx0zIQ,678 +jedi/third_party/typeshed/third_party/2/OpenSSL/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2/OpenSSL/crypto.pyi,sha256=N5TmTvFyT5KzQsoHdU7x2nU_G1Kf5TTQF1mFK-HBwMk,7588 +jedi/third_party/typeshed/third_party/2/concurrent/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2/concurrent/futures/__init__.pyi,sha256=zAZn69ohscdtL7rotnHyxSu7jzhVMw005ANG7J09apw,436 +jedi/third_party/typeshed/third_party/2/concurrent/futures/_base.pyi,sha256=rPSXJhb-LKP8qcFX9YQou0wCs33t6a3_hgfdD1zO278,3701 +jedi/third_party/typeshed/third_party/2/concurrent/futures/process.pyi,sha256=rGmQHTSHwVsXVPxwu4R1efuU0b8D1gqAEG9N_XH3vr8,195 +jedi/third_party/typeshed/third_party/2/concurrent/futures/thread.pyi,sha256=jEiKlhYG3p-MOOEpWFeYHNOKzK9vL7eAYEe-bI2V4XY,574 +jedi/third_party/typeshed/third_party/2/enum.pyi,sha256=iY_vEF2WHiMhmgZTtBGnE_XdzdPx90709cxW0BP9o4M,2643 +jedi/third_party/typeshed/third_party/2/fb303/FacebookService.pyi,sha256=ypViThxa9-z-_FsAfoHiXj9175tTBfqK2Vf2-8H7EL0,8692 +jedi/third_party/typeshed/third_party/2/fb303/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2/ipaddress.pyi,sha256=pkTNeeQfeOrdmRZd3aZanIWk4vgyW8bbDOfqrLrZvvM,5107 +jedi/third_party/typeshed/third_party/2/kazoo/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2/kazoo/client.pyi,sha256=fUPUW9-5VQibevcTDbpkhTkorz9JNyBfw6e03lrKgxs,3400 +jedi/third_party/typeshed/third_party/2/kazoo/exceptions.pyi,sha256=7DhQ3xMt63lVinoWQv0vwI6VmG2IfNPlMpLRGu_gq-s,2054 +jedi/third_party/typeshed/third_party/2/kazoo/recipe/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2/kazoo/recipe/watchers.pyi,sha256=Pxdwrx10R_orsLVINsoCsy-TL6a96EumRsENf2G11BE,551 +jedi/third_party/typeshed/third_party/2/pathlib2.pyi,sha256=Q9vqVVrQppWLvGeGATrn66__LyzByFxgWQ0maNUpYgk,4283 +jedi/third_party/typeshed/third_party/2/pymssql.pyi,sha256=vJAY2YPlF2arXuP2Yg05pFN8onxIqonFQD5uOqejAQY,1685 +jedi/third_party/typeshed/third_party/2/routes/__init__.pyi,sha256=JsiOugLeCiWHyhEHnCrs-i0H288m2ZaDpocEKY80qzE,364 +jedi/third_party/typeshed/third_party/2/routes/mapper.pyi,sha256=oBumS7A80xLzb_YA8ZEUnsqm5YjqE91r8m-9cw0lLlM,2362 +jedi/third_party/typeshed/third_party/2/routes/util.pyi,sha256=U9AvEa_p8Q1eq1zmIMKpGMjYqFbd9YSvTbv7W1Ez49Y,576 +jedi/third_party/typeshed/third_party/2/scribe/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2/scribe/scribe.pyi,sha256=2Yexj6Ymfe-Kj4_jigeicxrB2NEZryKUOtAqNN8Mm2s,1216 +jedi/third_party/typeshed/third_party/2/scribe/ttypes.pyi,sha256=Ub8_sHFFbDuMouo-wUU4bkr-WhUpSOVLBJ6zb9amqBs,383 +jedi/third_party/typeshed/third_party/2/six/__init__.pyi,sha256=L7AgaYaMdkiwH6Psg5nWG0MIvZ8gX4twwJ-Glf8zJqM,4390 +jedi/third_party/typeshed/third_party/2/six/moves/BaseHTTPServer.pyi,sha256=tGipj5ccgHt3EnzbxpPyQGUVnvu7RltrUOqgqWNGwpw,29 +jedi/third_party/typeshed/third_party/2/six/moves/CGIHTTPServer.pyi,sha256=omAocPqXteLJkGtdobdZjdzXynzCtn0jCFwBBlc3LUc,28 +jedi/third_party/typeshed/third_party/2/six/moves/SimpleHTTPServer.pyi,sha256=nC0fjp9wrhSnJMI8N2QD7ovWp1CW-dJWLyaV4K_Ql54,31 +jedi/third_party/typeshed/third_party/2/six/moves/__init__.pyi,sha256=6ipHwbeU3E61Sk4ZTAvfGuW7gr17r6zDZlINNDf2_4k,2105 +jedi/third_party/typeshed/third_party/2/six/moves/_dummy_thread.pyi,sha256=Va4LPdV5kK4luB-1NlkmycnVpXD4kKeYFD5QmmNqE8U,27 +jedi/third_party/typeshed/third_party/2/six/moves/_thread.pyi,sha256=9194lgQXs2W3sR00kP7Zn9gx9WWl0C49GEsbvzshDy0,21 +jedi/third_party/typeshed/third_party/2/six/moves/cPickle.pyi,sha256=hS7yIew9WYdHI_b6XwIlHAfRFRpUTKj7IokSZcpJ4PY,22 +jedi/third_party/typeshed/third_party/2/six/moves/collections_abc.pyi,sha256=Fcq1sut9OLQ824MQ69luYDJSzEB0LroJqTsXWgWGcDo,26 +jedi/third_party/typeshed/third_party/2/six/moves/configparser.pyi,sha256=r3G3JXE9Yo34AASn9AoNcielq9KuyQ3xrtElpnhRJYc,27 +jedi/third_party/typeshed/third_party/2/six/moves/email_mime_base.pyi,sha256=WcWEleCKHROrfdXpRuKABrT_Va1hx90NY_kxYeul3Sk,30 +jedi/third_party/typeshed/third_party/2/six/moves/email_mime_multipart.pyi,sha256=HRKWFU9qh95-mEE22_2NzEKL6lx7ynvhcfHjUcYWuZ8,35 +jedi/third_party/typeshed/third_party/2/six/moves/email_mime_nonmultipart.pyi,sha256=n5hD7R_rktJj3hiHYzEqr3CJCHSW4ikfObKHmUrXBw0,38 +jedi/third_party/typeshed/third_party/2/six/moves/email_mime_text.pyi,sha256=55VzBSQimrZf6UgotoXMiJDvqbKXly6-E_IXo6Ix22c,29 +jedi/third_party/typeshed/third_party/2/six/moves/html_entities.pyi,sha256=I0BI00vvC21L_BgnCbpjio-s1jqF4ARTt-qaol7mGig,29 +jedi/third_party/typeshed/third_party/2/six/moves/html_parser.pyi,sha256=hivJeBkqiAIZ6mvO1v4tOC9Mg6MzMR08P9tzsODdul4,25 +jedi/third_party/typeshed/third_party/2/six/moves/http_client.pyi,sha256=P8tgtt5Icp-ksHij6yPb_zuKk7ckcAHt_HM3aO0WrSM,22 +jedi/third_party/typeshed/third_party/2/six/moves/http_cookiejar.pyi,sha256=HUlF3MydQRX2Vv5G6KtN_Q6iCS48LBDggoDuPbEQUCc,24 +jedi/third_party/typeshed/third_party/2/six/moves/http_cookies.pyi,sha256=itzb5D5Mp66bx7hjyI3u-hri4h9jgqVzZyMfz4xNu2k,21 +jedi/third_party/typeshed/third_party/2/six/moves/queue.pyi,sha256=6Llng-UlZW_9HSWFgmIgW2q9YhaZ-Nzh2zJ8hkqoaZA,20 +jedi/third_party/typeshed/third_party/2/six/moves/reprlib.pyi,sha256=SWZYhGRU6moFAVBo5dUFUB9kyY6TO_kgrIqxzqDQ3C0,19 +jedi/third_party/typeshed/third_party/2/six/moves/socketserver.pyi,sha256=oeRnmecMYQfMmwRFVydatyCfs_HLrJYZvf5p7nm_ryE,27 +jedi/third_party/typeshed/third_party/2/six/moves/urllib/__init__.pyi,sha256=F_1V8NcR4jGkws85IUurYLi4JnGh7_HttdVHvj8cQZM,217 +jedi/third_party/typeshed/third_party/2/six/moves/urllib/error.pyi,sha256=73JoWrqvixKdnpT83GQ-6DsBc0gxqFC_xXN009qtCi4,129 +jedi/third_party/typeshed/third_party/2/six/moves/urllib/parse.pyi,sha256=Tv-Yji1tm-ikeaeLPXCcp1YVUa8y296SNV7YVupJtNg,744 +jedi/third_party/typeshed/third_party/2/six/moves/urllib/request.pyi,sha256=Z-EBXRE0CyHR8oOekYW6bl_w--YlvIVxkdbfcxtNKWQ,1453 +jedi/third_party/typeshed/third_party/2/six/moves/urllib/response.pyi,sha256=iSs21E6BAaB6P9CQngcuzBsGnGa9XN8P7P8GO2xYWjI,114 +jedi/third_party/typeshed/third_party/2/six/moves/urllib/robotparser.pyi,sha256=C8_E9lApZyMQpHflnHpYeyAgvQ_vFSuKon9Gl5DM3Q0,59 +jedi/third_party/typeshed/third_party/2/six/moves/urllib_error.pyi,sha256=7RTGNFpeUX5KEap9vyjA1Xc3Twfkut431Nu5290po1U,28 +jedi/third_party/typeshed/third_party/2/six/moves/urllib_parse.pyi,sha256=Q3BVGITL1UwlTmBsFD9iLf2pggJgTE5bG32QANdkMvo,28 +jedi/third_party/typeshed/third_party/2/six/moves/urllib_request.pyi,sha256=8WFe7ycArSuM6wJfgcXWLDRKNsymd0UlxWlflszb2yk,30 +jedi/third_party/typeshed/third_party/2/six/moves/urllib_response.pyi,sha256=dokFMleMVEVFVxBgSkrcn4f4yM7RhR3zkk0iDQGOC_U,31 +jedi/third_party/typeshed/third_party/2/six/moves/urllib_robotparser.pyi,sha256=8c26GW8MTI6cxDTD65N_18NRQcqWY4P9v8mrQm8c-oI,26 +jedi/third_party/typeshed/third_party/2/six/moves/xmlrpc_client.pyi,sha256=hL_FNiBles8aoJq0XQLbEHvWX1AedYbQopgRVQlbCEI,24 +jedi/third_party/typeshed/third_party/2/tornado/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2/tornado/concurrent.pyi,sha256=THaUUH9XW_opc-G528IhDDrf-cOBVx9ZJAQKAgoqr34,1016 +jedi/third_party/typeshed/third_party/2/tornado/gen.pyi,sha256=WPudybrtny-geS69g62Hkv-NApwTukIA8BrPCn5QzVg,2785 +jedi/third_party/typeshed/third_party/2/tornado/httpclient.pyi,sha256=FDo5peSsvOJFAfIxTgbvVqxk4zfsTxFYXcx7GEnDus4,3219 +jedi/third_party/typeshed/third_party/2/tornado/httpserver.pyi,sha256=YHoHZwbU43MoKk6EBgiWzu7OwD6jQTTdzjrIg4EbEac,1617 +jedi/third_party/typeshed/third_party/2/tornado/httputil.pyi,sha256=Ypkdy7WEucueAK8huNsEbp-SG5RoCiLuxzY2VFfYlbo,2853 +jedi/third_party/typeshed/third_party/2/tornado/ioloop.pyi,sha256=pO0QehC4_KN3eVtj7lnt4Z5PoCkzUT5CiTL6XxWsvyk,2798 +jedi/third_party/typeshed/third_party/2/tornado/locks.pyi,sha256=6LDrW8nuBZLsAe90h1UYs92k_41UjcxE8vpEhc_LBzM,1279 +jedi/third_party/typeshed/third_party/2/tornado/netutil.pyi,sha256=0u8D481OqDC_ZuSTQjFzWCmDmPp2H-xJuP2hsA2J8_Y,1350 +jedi/third_party/typeshed/third_party/2/tornado/process.pyi,sha256=NsnEzaGJtchXD3_1d5-cZ-C2q-h9ocucfLvT575cbrg,662 +jedi/third_party/typeshed/third_party/2/tornado/tcpserver.pyi,sha256=1N0cm3HUZlN3dfsgzrdYTNMEN3nvGu7r2UsE7WuJQ9Y,556 +jedi/third_party/typeshed/third_party/2/tornado/testing.pyi,sha256=L25CKrPr6bQA9NTQo5bDHfttn5WSAEwlky9y3vIINgo,1865 +jedi/third_party/typeshed/third_party/2/tornado/util.pyi,sha256=8eT1IqgZQAG2ffDTCdkAufpPxgtarCEPwbDua2Nuuqg,1072 +jedi/third_party/typeshed/third_party/2/tornado/web.pyi,sha256=3D7aMJ7GBsCFu8dho6qlXOnkklJxcPC37100DvJd1Wc,8848 +jedi/third_party/typeshed/third_party/2and3/atomicwrites/__init__.pyi,sha256=2rvz0VcbCzK5PZ_z1EkWACfHYm2-Lr68oowyr4P3pp0,850 +jedi/third_party/typeshed/third_party/2and3/attr/__init__.pyi,sha256=VDc3e_MSaIhknnBqIv-6XNjmDYR7pK2WCKydzh6D1Pg,8149 +jedi/third_party/typeshed/third_party/2and3/attr/_version_info.pyi,sha256=x_M3L3WuB7r_ULXAWjx959udKQ4HLB8l-hsc1FDGNvk,209 +jedi/third_party/typeshed/third_party/2and3/attr/converters.pyi,sha256=Bk1t2LtjazrzPzOBElUr6xTXwm1bSRT9yqX87xrEdeQ,346 +jedi/third_party/typeshed/third_party/2and3/attr/exceptions.pyi,sha256=4zuaJyl2axxWbqnZgxo_2oTpPNbyowEw3A4hqV5PmAc,458 +jedi/third_party/typeshed/third_party/2and3/attr/filters.pyi,sha256=_Sm80jGySETX_Clzdkon5NHVjQWRl3Y3liQKZX1czXc,215 +jedi/third_party/typeshed/third_party/2and3/attr/validators.pyi,sha256=ycA4ztBcRNQeniNpAENIu4BHB1RAyHgbZpS-8r8mwvI,1814 +jedi/third_party/typeshed/third_party/2and3/backports/__init__.pyi,sha256=CKeVKOomLpAIZH2ipts0a2F9ugydH1mCvfIGlfvtEbc,109 +jedi/third_party/typeshed/third_party/2and3/backports/ssl_match_hostname.pyi,sha256=jAdLMMvdxdszZCIdVGxRZw7i5waIBqb4RqwketJqMr4,81 +jedi/third_party/typeshed/third_party/2and3/backports_abc.pyi,sha256=z8KwJUh51HVvBluyv29D8TM7-fSQNZf_WPXGvddzV3M,220 +jedi/third_party/typeshed/third_party/2and3/bleach/__init__.pyi,sha256=M2COlKhdPdtUHIMaOWtlGIVk5NGzag1OEEcXL0vtTo4,841 +jedi/third_party/typeshed/third_party/2and3/bleach/callbacks.pyi,sha256=Wpeg0LxBAend5GwNZVw0u6dKT4ZoicVDrUqsGqFJMFs,206 +jedi/third_party/typeshed/third_party/2and3/bleach/linkifier.pyi,sha256=vQleEwvJ8ucOi0oyBhRnOXj6dAGZe_2bHRHX8UqRFPA,978 +jedi/third_party/typeshed/third_party/2and3/bleach/sanitizer.pyi,sha256=1-c_WuvDFDEVINySuZEuaBq_RfYjf7UV74b9pq-15YQ,1148 +jedi/third_party/typeshed/third_party/2and3/bleach/utils.pyi,sha256=RSxKxwsKbaak2b50U5iFJsnkMGQDSEADBN8fqamTZGw,286 +jedi/third_party/typeshed/third_party/2and3/boto/__init__.pyi,sha256=daXgcCm5ezx16EpC83J04doV5fHQ1mZlLLCEgIltRZY,7183 +jedi/third_party/typeshed/third_party/2and3/boto/auth.pyi,sha256=H6ZtG0lKjp4WCcoUcNxecLo4CzDs1xCsUYmwI3yeSYg,4141 +jedi/third_party/typeshed/third_party/2and3/boto/auth_handler.pyi,sha256=7QChS7_s-Uit5ZNCr4FCtMRFRw7uOIJUUW3ISeAckXQ,251 +jedi/third_party/typeshed/third_party/2and3/boto/compat.pyi,sha256=v6PUjkCJ0tdXWWUz-0vx2vSFBbh0pmk3yfY3EFhLg_g,384 +jedi/third_party/typeshed/third_party/2and3/boto/connection.pyi,sha256=g7eTL8QplrrRBsQoGhIEEE4y4puNzPFQDuEDXq1o4bc,5679 +jedi/third_party/typeshed/third_party/2and3/boto/ec2/__init__.pyi,sha256=mJotwEhqx0uTOkp7ABAXmgruW432cmd_CLfyqwIdh7c,256 +jedi/third_party/typeshed/third_party/2and3/boto/elb/__init__.pyi,sha256=vIwGOPGo2wMeNQLxYnBTI5nq-35Q2bcZCyXmgK7WfGA,2629 +jedi/third_party/typeshed/third_party/2and3/boto/exception.pyi,sha256=pp1-GMr4VjeXHcmHJsiBvyiYC5I1MN6iHV_kclA1Mg8,4569 +jedi/third_party/typeshed/third_party/2and3/boto/kms/__init__.pyi,sha256=kblf2ubHjX319n44aDeqTcaZmNGcE1aCNVX1_eprqM8,157 +jedi/third_party/typeshed/third_party/2and3/boto/kms/exceptions.pyi,sha256=WKhoeQKeApIczWPJz3vC5A7K03AmbNAM4bnBuVuNVCw,829 +jedi/third_party/typeshed/third_party/2and3/boto/kms/layer1.pyi,sha256=OgMqR0jIffdQCX7bqbN8WybQrfrr6sXj9CGvNUC_cHA,3972 +jedi/third_party/typeshed/third_party/2and3/boto/plugin.pyi,sha256=rquBCp_gv_wSNNcWYijFPURhXO48ELzf5XItCuPv7nI,235 +jedi/third_party/typeshed/third_party/2and3/boto/regioninfo.pyi,sha256=QkiqbCcldisKPERnXU6fgKYlOHD4ojOWssGq87r2KMw,701 +jedi/third_party/typeshed/third_party/2and3/boto/s3/__init__.pyi,sha256=9lKx0pFB5gzA3515idyUeg6YF8zryDGfm1hWzOWlSbo,523 +jedi/third_party/typeshed/third_party/2and3/boto/s3/acl.pyi,sha256=01T9Ak5fcjt4TGf2vpCW-T0rS-miWRpcdKttcYBBZhk,1686 +jedi/third_party/typeshed/third_party/2and3/boto/s3/bucket.pyi,sha256=576y7ybFKFnWpxcFeYHsmCg1iXsy0fv3BST2eIbEDG0,8747 +jedi/third_party/typeshed/third_party/2and3/boto/s3/bucketlistresultset.pyi,sha256=3UER4BACEqM1ZpY6P3kefwwXuBP2TxD8UoqMhPbWtl0,2029 +jedi/third_party/typeshed/third_party/2and3/boto/s3/bucketlogging.pyi,sha256=3A-6a-73DgObYEzBbcUcknHJQyw4e3ihu0NDL4qRmGQ,400 +jedi/third_party/typeshed/third_party/2and3/boto/s3/connection.pyi,sha256=pkr9v7Bxd0pyu-IA_bkW-ueRCTb5uzed_UT8G_Frp-g,4829 +jedi/third_party/typeshed/third_party/2and3/boto/s3/cors.pyi,sha256=wrlgEU_VFds-ZtwzNG00aCwil2hZxRaSaBkMq6yGKo4,1088 +jedi/third_party/typeshed/third_party/2and3/boto/s3/deletemarker.pyi,sha256=T6AWOwNnW391UjVpzJ8tfbaOtM_hSaNa2wyJV02f5wQ,366 +jedi/third_party/typeshed/third_party/2and3/boto/s3/key.pyi,sha256=73WWtWTIbSJl_vHm8DqYCi2DCa7IuVsCq07I4TKtw-g,8317 +jedi/third_party/typeshed/third_party/2and3/boto/s3/keyfile.pyi,sha256=EcfBooXpPExAghucYEM6sxPVsp32wueqK_-5Kk-66rI,684 +jedi/third_party/typeshed/third_party/2and3/boto/s3/lifecycle.pyi,sha256=eevvO0p0sW2lQYzyxVGUgEe7yLDsnia-Tnqq0ibgwU4,1984 +jedi/third_party/typeshed/third_party/2and3/boto/s3/multidelete.pyi,sha256=GpNwfrASD9rWAfPhd1HoZf-6rBT2Lf3bbJyj_8fX4Kg,1075 +jedi/third_party/typeshed/third_party/2and3/boto/s3/multipart.pyi,sha256=G1A4EP8qX15z724aoTc-13cAZnuSl2V-2Gbp4GjlPjA,1997 +jedi/third_party/typeshed/third_party/2and3/boto/s3/prefix.pyi,sha256=UDLPJn1XiSFLXxd0RdeOMzUfzEyYUnCAiGPACntnkkI,324 +jedi/third_party/typeshed/third_party/2and3/boto/s3/tagging.pyi,sha256=JgNFPHTLi7TR0HbeasYlSnbVA-GzNXzYJ9H6sBmLz0c,748 +jedi/third_party/typeshed/third_party/2and3/boto/s3/user.pyi,sha256=nFbH3dOf5SfNoaTyNwkiu2q3ZR82iPYVr-LfXzUGzYg,362 +jedi/third_party/typeshed/third_party/2and3/boto/s3/website.pyi,sha256=O-9DSWitjeUMEkGVnCUzOQjEOuX94HDt_9bPgLEnkBw,2649 +jedi/third_party/typeshed/third_party/2and3/boto/utils.pyi,sha256=9iLNLIEyh6Z5rbXpdGef7V5eQ0jih7eSA5sEwgtxAOI,5876 +jedi/third_party/typeshed/third_party/2and3/cachetools/__init__.pyi,sha256=i7v_tIZJRWRJCFiADCwVOOp_t9Vo_aZp_cEL0lcOBJ8,254 +jedi/third_party/typeshed/third_party/2and3/cachetools/abc.pyi,sha256=IqWLRykYAB0tVeDGtxAv4hEBeotGQaxIPkrSMYWdlAg,182 +jedi/third_party/typeshed/third_party/2and3/cachetools/cache.pyi,sha256=xnBVZ9h9R56nhR4vWfH_MAXev9KnLLtEqLN4405zQww,712 +jedi/third_party/typeshed/third_party/2and3/cachetools/decorators.pyi,sha256=jao0eXl3rjETbO5_TUtpRNiHg83q6MqK-YPtshaldk4,605 +jedi/third_party/typeshed/third_party/2and3/cachetools/func.pyi,sha256=p4rD-rrZVbHD3x0HVrmQf2Sv6gCqDxqo5Ln-AGCnfSc,510 +jedi/third_party/typeshed/third_party/2and3/cachetools/lfu.pyi,sha256=hEuvwO-6VmvZdl2bkFWuP7yq6jmCeB2Z9HVPC-vC9Rg,632 +jedi/third_party/typeshed/third_party/2and3/cachetools/lru.pyi,sha256=1PkODHl1pNoGmDhZEUh0qvQQLDZKUJTfKPhjhuT73WU,607 +jedi/third_party/typeshed/third_party/2and3/cachetools/rr.pyi,sha256=f76mZgCsS7a8K44HFchyia1hAAAAxDIg6pdlVhfOQ-g,683 +jedi/third_party/typeshed/third_party/2and3/cachetools/ttl.pyi,sha256=I-P_gSblYWVijxd6WIFFAYEuqf6qI2wApq4RuFxekPk,926 +jedi/third_party/typeshed/third_party/2and3/certifi.pyi,sha256=L-Idpv54Z5MALmou6rENZOu7F_jAFD5CPo1hb5amF2s,24 +jedi/third_party/typeshed/third_party/2and3/characteristic/__init__.pyi,sha256=3h0tr1yazsqIaoo23raa-kUVvvfFTzzsMHllLmJBgY8,1330 +jedi/third_party/typeshed/third_party/2and3/chardet/__init__.pyi,sha256=iTl0aDlS_uQVPCs4ZQSsAYwNveIOYkE3ADM0Bu09Q04,667 +jedi/third_party/typeshed/third_party/2and3/chardet/enums.pyi,sha256=inSdRKDA-1mWbD2S2Wk1NRmWRmefiYx5YQxJMGimkz4,710 +jedi/third_party/typeshed/third_party/2and3/chardet/langbulgarianmodel.pyi,sha256=OpF1fUvRwXdQXUXoJs33oTP4KRMWZF05xwZjkLu6iyc,263 +jedi/third_party/typeshed/third_party/2and3/chardet/langcyrillicmodel.pyi,sha256=QbAYnXFTbOS3n1OntaCWI1x3XIbWX96oRwyyA9A5vZw,536 +jedi/third_party/typeshed/third_party/2and3/chardet/langgreekmodel.pyi,sha256=T0VzNfG5ktCgMT8PY-JSmVToVUXYxY9T72iaYQ9xmmk,240 +jedi/third_party/typeshed/third_party/2and3/chardet/langhebrewmodel.pyi,sha256=tUeIdTVWsWmI-RJAIkk5ubiUqeI9R1vUElAk1cdCxP4,169 +jedi/third_party/typeshed/third_party/2and3/chardet/langhungarianmodel.pyi,sha256=6sMaBbtCipjI9zwb3pK2ojnJ1V50ueO7cLd3ODuBkVo,263 +jedi/third_party/typeshed/third_party/2and3/chardet/langthaimodel.pyi,sha256=X0Ot8G3kPmA4VrnAxy7wwAkLyZJ5QfLOQMMSZfDpNH4,157 +jedi/third_party/typeshed/third_party/2and3/chardet/langturkishmodel.pyi,sha256=0xxalkd4p4lrZFWdCy9OZhvAG_S1rPEH2HuguWFe_7o,171 +jedi/third_party/typeshed/third_party/2and3/chardet/universaldetector.pyi,sha256=Cv9ikp-HF1JsbmUUnfrTGdj-ogw_iFVSf0yOQEE5Hxs,809 +jedi/third_party/typeshed/third_party/2and3/chardet/version.pyi,sha256=9L2O530uTJPPoHbmYkRjZwFkoIeIOhCWwGGBYouCKN4,61 +jedi/third_party/typeshed/third_party/2and3/click/__init__.pyi,sha256=3oe1-v8WfI-v6asPrtdIp1ochj2vT_cKOt4W6PBUx0E,2273 +jedi/third_party/typeshed/third_party/2and3/click/_termui_impl.pyi,sha256=gMqbhjSvNlAmi6GaFanVip40LRi4taeSN4sR_ck10II,442 +jedi/third_party/typeshed/third_party/2and3/click/core.pyi,sha256=lRB0bp9srrgM9TRZ8ZKRV71c2QRF1EOi2DBE7ePclQs,11012 +jedi/third_party/typeshed/third_party/2and3/click/decorators.pyi,sha256=UbSVA_xm2O6BZhfl-K0Xi4a_xzuMiby4_pz9gZx2X8I,9143 +jedi/third_party/typeshed/third_party/2and3/click/exceptions.pyi,sha256=o4rbUXi-6iBjHNiHUjnEo-Ad02Dy3xrZL_YYPVkWN78,1936 +jedi/third_party/typeshed/third_party/2and3/click/formatting.pyi,sha256=D6GMWwb6-9mM7-F7Om4bQP-pbtFKD2NZ_rDUKYNpjvA,1383 +jedi/third_party/typeshed/third_party/2and3/click/globals.pyi,sha256=Xi3fHtTEJHjVyQ-djJEaQkDxnZ9RE5xD-uh2B1xs9ho,274 +jedi/third_party/typeshed/third_party/2and3/click/parser.pyi,sha256=tSKZ0bKLpIObxaSZ86IBpk7PsR6idJGXD2bZzgya5ss,2006 +jedi/third_party/typeshed/third_party/2and3/click/termui.pyi,sha256=cG1FM90WpFvr8VQ9-XY_zstLebtXRK2MLZ0ncPqwMXg,3291 +jedi/third_party/typeshed/third_party/2and3/click/testing.pyi,sha256=s1GKTPK-blyfhaXqk3NvFBAXRBGuCR73RRH_weGbOcs,2136 +jedi/third_party/typeshed/third_party/2and3/click/types.pyi,sha256=VSx6kUdgT7LHTW2Ig8-pvnD9t8AxUYYSYUabbpsDdyM,5045 +jedi/third_party/typeshed/third_party/2and3/click/utils.pyi,sha256=Rgm3qjXceoORj5TgL7fHcBMs5bU6j2EU7qC68ZaVElU,1752 +jedi/third_party/typeshed/third_party/2and3/croniter.pyi,sha256=TJBwRUQdiVQbCYBiao1VL2Esal2U_I3n6ZR38NtOd7o,1934 +jedi/third_party/typeshed/third_party/2and3/cryptography/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/cryptography/exceptions.pyi,sha256=CpK5HtZvigk1U7yYLlqYcXGJaOzknyfhUUittXFuKiM,262 +jedi/third_party/typeshed/third_party/2and3/cryptography/fernet.pyi,sha256=pr8Po1ra3EdG9lMU8osc1fg0-ZOoASGAHtccbhcbr80,1282 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/__init__.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/backends/__init__.pyi,sha256=dKFUFhsHmSbNrHdG2a_5CErfvQu9hdnIWMA1vys9LaI,124 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/backends/interfaces.pyi,sha256=B6ll_f3g9smOyhOZR8hRLJcc891y2yr57KPSVQYb1PI,8238 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/bindings/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/bindings/openssl/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/bindings/openssl/binding.pyi,sha256=QHIbjl0Q39fxas4DabuLytXjjBZrfqNkft0H_nWwXr4,148 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/__init__.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/__init__.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/dh.pyi,sha256=AHR65ed03NWB6oK8YoTQYt3AgCk0fMk179NRcE5WfmQ,2651 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/dsa.pyi,sha256=AeKPk-Evk0aTJufLg3ohlne7qR2CXJqMse8gNoAD-y0,2965 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/ec.pyi,sha256=VoSyXBfG2NuAqHd_WARe7UH1eqq9jaO5vijRMNk3AFQ,6340 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/ed25519.pyi,sha256=OiE4CJcNDIaqd_VzdmQGn28WPdT5PBQ5jVKfq_ZRc3k,985 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/ed448.pyi,sha256=Mw-p07GDyttBXKaNB5Su5hw01DQXZgjEZe5M1e7PlhE,973 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/padding.pyi,sha256=g1sB5wpH6yoi9LrLUSHOejKdt8xiLmkPvDdUn6Cm1UY,787 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/rsa.pyi,sha256=rT_qcbpenlB8wuhg3jRcMm-5OErh2cCc3tkNFn-x1d0,3288 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/utils.pyi,sha256=nTFxj83DVi4OEi8B9wokES2Vh-HwUsxZHiEA4PcH068,406 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/x25519.pyi,sha256=JHX8lIJN0j94AsHIS4jQWCrlcOgLdb3Ud2QznMRXOC0,919 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/asymmetric/x448.pyi,sha256=efSTvyRkpVAcxtAwzYymDbnOQPA-dJW7YAliRVA5_cc,905 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/ciphers/__init__.pyi,sha256=eOEIBpUXLaH5IkOca5Ko3HQdUGfDIDFDwmlsz7RBGJA,1363 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/ciphers/aead.pyi,sha256=MhkfsUnaMVEi6oRrcfTgp6h1_1QCS8bvxtwoE6bqZS4,1065 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/ciphers/algorithms.pyi,sha256=Ig9mWCRz8rM6X5js1D4rlT06YbdALJ5RUpO4i6Qb7Io,2245 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/ciphers/modes.pyi,sha256=3rIwa-kj4k0wj8YfZeesqX6jEJhHtUZth6-ivi76etE,3089 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/cmac.pyi,sha256=HO4SJ5cJgPcP3cBjXg6n8nAKBWpyOlQzyXZfyQz_af8,461 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/constant_time.pyi,sha256=47ogru9j6XG5mLMDsl2iGKxhZjsH_bKhWYBt4c1jQk0,46 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/hashes.pyi,sha256=j3L6NIrR867bxvIoY6FVw10HEZz-0alVzDTUdPqf48w,1342 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/hmac.pyi,sha256=4iDj3W1Nk5MKJdkpNVRsY3Q_KP9NBOK_zS7pbo1Wj_0,457 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/kdf/__init__.pyi,sha256=fymJGVYF4JGsuO5IUacyuYb4Xunrps4mFrJ0XFEUdfU,261 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/kdf/concatkdf.pyi,sha256=Qmc9PP5j2fsiJfQltiWvgywgNLPQVTcyofH2mNqmWmI,951 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/kdf/hkdf.pyi,sha256=m_JHDA0r_aZGcOT5B0_3H9VpNfN9WnEAvb1zvgIssa8,902 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/kdf/kbkdf.pyi,sha256=MCUGyI5wZWLcPd9Sh_aHnDfumqS9ZoCYsQ2WseK-BM8,867 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/kdf/pbkdf2.pyi,sha256=-MWdJfLGEKiNC3qavnm2unJDtndsBFmZmQoj4Cbi_N8,561 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/kdf/scrypt.pyi,sha256=wKA2YyDAullLwwfR9Ng43xn2gKd7GyCH9NGVSRz1Ycs,452 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/kdf/x963kdf.pyi,sha256=wvi3YJDqDJdqVT-Dgr84e4-cqosiq2f_wQQD6mgFXsk,545 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/keywrap.pyi,sha256=FGJg_GstrQcaavqriIfHdq_aOQPVeq_F9CpN8Nw0gvI,611 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/padding.pyi,sha256=RYc1U9k5Vw4NzpT-JXi_AwRiPXwzu7XLVyG6HtDy_gM,540 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/poly1305.pyi,sha256=ON5PxE5sYR6YLf8grWy-8FwUfGiQt28l1s2elvrKgYw,375 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/serialization/__init__.pyi,sha256=7RHfNmeDX9gwtceJXJ7kdp_YCpqHB1BZObkeDk2YSAI,2359 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/serialization/pkcs12.pyi,sha256=NwgQSXWYnaOm4717c2v92AP8jrw0davzciJDVzeDIjE,933 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/twofactor/__init__.pyi,sha256=MV00ZbBi-FKgHi9PYY2ZS7tneV2rghu2eVi1KgxRZzs,35 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/twofactor/hotp.pyi,sha256=ibl6C1ISmQ5H6nmmT_axuSuhPmPUn7YBx7A4i5PoFbY,540 +jedi/third_party/typeshed/third_party/2and3/cryptography/hazmat/primitives/twofactor/totp.pyi,sha256=p2rGTlM-DzrBmqLnYQKklWHjJJbDwQ0yf_J1BfJqf8g,585 +jedi/third_party/typeshed/third_party/2and3/cryptography/x509/__init__.pyi,sha256=3xR6GVwKT2nPtrlazcfVcs_r5DmEFkYk1OnstSBjiQ4,17734 +jedi/third_party/typeshed/third_party/2and3/cryptography/x509/extensions.pyi,sha256=zMbVNkp-hl_XW3RW71h0MlwUM-Rrz3X0OxEEZbZAo-Y,557 +jedi/third_party/typeshed/third_party/2and3/cryptography/x509/oid.pyi,sha256=eHglWkt4t9CRxOwXqdsocMPa8MpJFTHn1lhBf5EAfMU,4153 +jedi/third_party/typeshed/third_party/2and3/dateparser.pyi,sha256=xAhfYbQgduxqvMxoMSfuaBeQ5pzwLWVrZTdbOzHWMYo,522 +jedi/third_party/typeshed/third_party/2and3/datetimerange/__init__.pyi,sha256=_CLVxkVpVy20mXcnlMv8KzMcvSEbnEKgYfScf_swPuk,2062 +jedi/third_party/typeshed/third_party/2and3/dateutil/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/dateutil/_common.pyi,sha256=Xxs-IpVplkwjwEnqCLwzA-JIm6I9kzAFkcLQWyzZWvY,340 +jedi/third_party/typeshed/third_party/2and3/dateutil/easter.pyi,sha256=5mypiQN0QZHpFBk80RddMUkFuRHb4qWOX8wJCbx4a6s,214 +jedi/third_party/typeshed/third_party/2and3/dateutil/parser.pyi,sha256=Brz_8xe_yX14RB6B8pVk6BacM6HizqkUCIQ5vc9U-_I,1779 +jedi/third_party/typeshed/third_party/2and3/dateutil/relativedelta.pyi,sha256=SfCOq1b8otE6in4nsnH0m0Kbyl2jbWaxGwcBoeapTHk,3243 +jedi/third_party/typeshed/third_party/2and3/dateutil/rrule.pyi,sha256=EBup_m5smahe2tbtJG_-0kj6pyZjyv0YKi5oWKjwP1s,3219 +jedi/third_party/typeshed/third_party/2and3/dateutil/tz/__init__.pyi,sha256=MlXCZJGqduJh-Mk0pHLw06oNxeW1fvPP3ixfo2SM8ak,340 +jedi/third_party/typeshed/third_party/2and3/dateutil/tz/_common.pyi,sha256=0C_pqZ2TLRyKAsg_lS6WR-pwbGiezi4PY7uotzfUMjk,818 +jedi/third_party/typeshed/third_party/2and3/dateutil/tz/tz.pyi,sha256=mExYydbsNJYcED03KVMfEHthqixakuSa3Qq_9mDczUc,3970 +jedi/third_party/typeshed/third_party/2and3/dateutil/utils.pyi,sha256=c9oLGnocofsb4y9CPpCIZGPFF_tysaA6P9i_g3CpZVE,281 +jedi/third_party/typeshed/third_party/2and3/decorator.pyi,sha256=MXFxKax2ovqI9ZAJRmWIyNzOxL1sYQ2vGBM6JGkl_44,2760 +jedi/third_party/typeshed/third_party/2and3/deprecated/__init__.pyi,sha256=Q5NJtAoHP5mH31hhagi9d8-jI3qoRJDlRQNCMMBpOG8,46 +jedi/third_party/typeshed/third_party/2and3/deprecated/classic.pyi,sha256=lWfx7eJTqKQPcZr3oKLHHrPB29S-vYm8yImwb3enRM0,783 +jedi/third_party/typeshed/third_party/2and3/deprecated/sphinx.pyi,sha256=NvfjRlHcPAeTbI_BkSg9ygv0klEMxTShqP9i5GjfaSQ,1129 +jedi/third_party/typeshed/third_party/2and3/emoji/__init__.pyi,sha256=Ix7SRTSbCqLgmR0L0MCBIW72Zs0t3-mmmbdMzg4Axbo,373 +jedi/third_party/typeshed/third_party/2and3/emoji/core.pyi,sha256=-4Hs3zO0KZtnou62-FII7AZbyEpL__-VRSziFn8acTo,419 +jedi/third_party/typeshed/third_party/2and3/emoji/unicode_codes.pyi,sha256=qwYCvETNLt20Ok76oiBBIuVfIzzEkdLt1jhxGzdqHUA,171 +jedi/third_party/typeshed/third_party/2and3/first.pyi,sha256=jk0nVWOV_kqCy1ZIvuTPGbggVajhW9SnGVYdDflfss4,481 +jedi/third_party/typeshed/third_party/2and3/flask/__init__.pyi,sha256=dKC5xsUiM9a0nh1zxuP991HYODlG4Th_-2_uifzieFI,1688 +jedi/third_party/typeshed/third_party/2and3/flask/app.pyi,sha256=E352cSxQISVJ894PKmp07vTwurIyadV2RRI1Cro7m-Y,8224 +jedi/third_party/typeshed/third_party/2and3/flask/blueprints.pyi,sha256=g_ZbSf5GoDmzEbTyJUnDydPwfK--8soIOApyjCgjg3w,3564 +jedi/third_party/typeshed/third_party/2and3/flask/cli.pyi,sha256=WwdlCr0pd8tNj8W8B2XHg6jIlh-ryn5vzaBvvWe2b3U,2195 +jedi/third_party/typeshed/third_party/2and3/flask/config.pyi,sha256=KjCQX-yxie9WA0K5-Zujen_kja_GyptfYqTw_L-QdhM,871 +jedi/third_party/typeshed/third_party/2and3/flask/ctx.pyi,sha256=JJ9QMsJoIWCudf9Gwi_9PmFhLHh3Qi-m50IklJOrnOg,1335 +jedi/third_party/typeshed/third_party/2and3/flask/debughelpers.pyi,sha256=xW9zO6fuhvTJ4rk-tQFWucJOzufOJjHaahF7_Hm4HYQ,507 +jedi/third_party/typeshed/third_party/2and3/flask/globals.pyi,sha256=C86I2edNhxlipXPH6Ghwf2gN2E0ALtd7uRe2f1A5NOM,322 +jedi/third_party/typeshed/third_party/2and3/flask/helpers.pyi,sha256=2sriNHIKIlknrelbfl2Kh9Fd-Gzqo1qX3G0pptqn5ig,2036 +jedi/third_party/typeshed/third_party/2and3/flask/json/__init__.pyi,sha256=1UCkfXT-tCvt4MW1loZM4fXMd9_8U91nYZyMCzddUSY,654 +jedi/third_party/typeshed/third_party/2and3/flask/json/tag.pyi,sha256=_cOpm75btUJ-zPPdvBeGxj12fENH_9XY0_I32Ounk3I,1918 +jedi/third_party/typeshed/third_party/2and3/flask/logging.pyi,sha256=AOGLuCAg9HnXXkXQsVGa2lQ6TAxqk18G0RpytsD0SBo,150 +jedi/third_party/typeshed/third_party/2and3/flask/sessions.pyi,sha256=rrIKmlljalXzcTNH_x5YVEVsmMKuIJnJ1r7XHI2RTfo,2017 +jedi/third_party/typeshed/third_party/2and3/flask/signals.pyi,sha256=HRUu9jh583VLFfxUmdNjJ17XnVdziRDu-_9NIgnf2yc,718 +jedi/third_party/typeshed/third_party/2and3/flask/templating.pyi,sha256=xBS-FLM_qPjNQwx5Z5a-c48hQ77vfC5LXLNCJCSmI4g,604 +jedi/third_party/typeshed/third_party/2and3/flask/testing.pyi,sha256=oiZz2eCxXMLPjChHuLps-2HwtApsmWgY0nU1fFmXc74,1827 +jedi/third_party/typeshed/third_party/2and3/flask/views.pyi,sha256=AXG5OxY9r03ZDSFtV48yXsFPXQlMFZtD0cHceIpwzIg,530 +jedi/third_party/typeshed/third_party/2and3/flask/wrappers.pyi,sha256=Qhblio57KSJw-stcPOYTRxCWtRZ7V7yCCMtz8q2JkBc,1145 +jedi/third_party/typeshed/third_party/2and3/geoip2/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/geoip2/database.pyi,sha256=KiGpw7XqZHu0bSQSFNDu9yW-RDbP8XGTQCD0LTO_soM,1133 +jedi/third_party/typeshed/third_party/2and3/geoip2/errors.pyi,sha256=pZ2Y9djJdVQVgKMnPXPI77UKCF_gs6v9ApuWSj4tgSY,494 +jedi/third_party/typeshed/third_party/2and3/geoip2/mixins.pyi,sha256=ZpggdVI7qLOYVHt91mL8uPwWq4E07MxpN_UfAgoPcB8,120 +jedi/third_party/typeshed/third_party/2and3/geoip2/models.pyi,sha256=h0flgP22m1KL7U55cg1PAhoQ73XeGPg5s2nLn6B_5aY,1867 +jedi/third_party/typeshed/third_party/2and3/geoip2/records.pyi,sha256=3-gke4nBlwnk09iOiT8tRdusFPi6CewrJn9foXKvI_M,2071 +jedi/third_party/typeshed/third_party/2and3/gflags.pyi,sha256=c32SjcBY5EaEWC0ZzMWqfEM9BfMa8sMc4nX_WntL3fc,10776 +jedi/third_party/typeshed/third_party/2and3/google/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/__init__.pyi,sha256=U54VfsG7-8DG_1f27w_Gmz8NunTjIR0iCWEXUCuKKVA,19 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/any_pb2.pyi,sha256=r12boNnMlrqzwGMOVLK3YUILTg26UVP3uGOav2S3vgo,1353 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/api_pb2.pyi,sha256=Vk1syiUbOtrQuhC5HKZX4RPYKtdAbypTsX-fZox8Gtc,4881 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/compiler/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/compiler/plugin_pb2.pyi,sha256=MhJF46Zpl7YfIhQyUBXmG5pkOYIww3CSrkhyIK_FL58,6548 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/descriptor.pyi,sha256=fkv0RR_gPgZr9EJS7tisfkMQtiYDX8OOCGeY7DKBSvc,7843 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/descriptor_pb2.pyi,sha256=A7YzWInNH0SSQrlE4wXAw2vbnCNw93zkyu196Gm8rto,43146 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/descriptor_pool.pyi,sha256=dsqeewyqydUwYE9sNscI2qvYE1h5huXeZQ9igxjQD8I,744 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/duration_pb2.pyi,sha256=jBEaeqcqCopxOCSWjbhP8zjXLf-VkCBFSZeZ1G8I1UU,1348 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/empty_pb2.pyi,sha256=FWX0qLZZMDX34-I45EwCjvOWnNdvwGJ-yE3DcZXoZJc,603 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/field_mask_pb2.pyi,sha256=6E26cc872vDtSCX0B1XQGAHUW_ajwYcCtKKg-jrejJg,1558 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/containers.pyi,sha256=FRcEQWw5vvdjZ5fD80mj9G_9tdZjOTgWiMGIoV6fhnc,3653 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/decoder.pyi,sha256=roJK89CkgjVKjRJbEnQ5ik1KdfXhwxymSyRIg7c5WW0,860 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/encoder.pyi,sha256=3rRgqC6ceJ7LtL7Qo5YiAK6CwE_HigHMsj9l-oa_5iE,1045 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/enum_type_wrapper.pyi,sha256=HPVeZHGr2DNnMI9bX8hRNii3OrD8DtWIr2fawf8BQ2Q,650 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/extension_dict.pyi,sha256=e54-8z3JwpIKs36-NI9VWfQ9LEncwDMCXJrJI0wint8,1795 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/message_listener.pyi,sha256=EiRlRR-kzJlKV3NXoYtYrGhMS3lZ2oWwQdq77h1cY8E,148 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/python_message.pyi,sha256=IESr1OjAZD9idg1OgWzMfeV2Sta08R0uHLAbweljTD4,46 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/well_known_types.pyi,sha256=DFxcLR_wHeTGdA49Vl-dr8k0zdm7LlksB6Dqj3ZD5Yg,3756 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/internal/wire_format.pyi,sha256=ncrIlZeBjytfWeG9rQX8VdHO8YvS3EmqBJ4rIbaZVxw,1554 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/json_format.pyi,sha256=ZCsnQy2nbUTYAkOHilhoGAu0gMyiTGx7rD1wTMEH75M,903 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/message.pyi,sha256=wxYo7wptWBT2maYMBCMwEGIyuTrPzmjUNmGOly4wmhE,2522 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/message_factory.pyi,sha256=LOP5EeGqIi8j6R0V-4eoG9iZNX30c3dQb1J59td_4Ks,631 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/reflection.pyi,sha256=VahfyfCz4LzllaCNcv6qXVMa6sWQZ9PEbM9yTKrC9d0,230 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/service.pyi,sha256=khFSQ7jbdMgCZRnhTDDIXnaDvS_GsjhllVxiJpDu9yg,1371 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/source_context_pb2.pyi,sha256=ojA6bEP8HWs-2i-UH8GOxc8z3fYZGq6diaj1vZLTEgU,1097 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/struct_pb2.pyi,sha256=g1CuFe2Cd37bigkRiOudt2hoIA27LIH0YcxxM1iym1w,5358 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/symbol_database.pyi,sha256=skHeJKelHAiApzBN2EKOd6VaWP3IsyjOxoe0HDS3X84,912 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/timestamp_pb2.pyi,sha256=3WwiKOf2BEcFMLwe480tAQWPDG1xRtRebgwvj2xIN9o,1354 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/type_pb2.pyi,sha256=TNdAmPxbIYjLzTc_4nMSy3tyBY3gPN7Vteotfh6aNxM,11139 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/util/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/google/protobuf/wrappers_pb2.pyi,sha256=rew4YgJVRs7PuxNVZggdHPISr2QuNOYmDbj2gmZIv8k,4287 +jedi/third_party/typeshed/third_party/2and3/itsdangerous.pyi,sha256=ufy10z6xR-SpjLpa8c89G4uN-vu_LGaGGiSMzA-PtYw,8405 +jedi/third_party/typeshed/third_party/2and3/jinja2/__init__.pyi,sha256=FGOBNYKOz5GYd8TgZutpo9IKAZ32BiGnbkiz1dWTIpg,1529 +jedi/third_party/typeshed/third_party/2and3/jinja2/_compat.pyi,sha256=7I95GYBNx-7EWPd3BvePzanb1IWGiOJZwAW2AdCJLwQ,598 +jedi/third_party/typeshed/third_party/2and3/jinja2/_stringdefs.pyi,sha256=_gw2yhYfWIlqy06ve5t93z3SIVXX2j5Md3rz58LIKjE,360 +jedi/third_party/typeshed/third_party/2and3/jinja2/bccache.pyi,sha256=ypfXFrP5k-Phe6dJ3jLwn3yhRMzDSYPHTpsJT4Hx56M,1396 +jedi/third_party/typeshed/third_party/2and3/jinja2/compiler.pyi,sha256=PGyzLL8q-_nQVW45vTCeyclt-UuWjln6UqewcHsju9E,6363 +jedi/third_party/typeshed/third_party/2and3/jinja2/constants.pyi,sha256=seJNAMX1_sGZ8wi8SB5qbZtSUvYiQMelMpXMyxl-AR4,23 +jedi/third_party/typeshed/third_party/2and3/jinja2/debug.pyi,sha256=6XZeMpZjfFUP-Mh5tEof8P8UANNgJ7lsBq1RUbjL_M4,1018 +jedi/third_party/typeshed/third_party/2and3/jinja2/defaults.pyi,sha256=fe4KjTE3_Kh0bG7yXzSy6cqtlkHTsRgMWYyMzLnBsag,532 +jedi/third_party/typeshed/third_party/2and3/jinja2/environment.pyi,sha256=7cYxhgLusiW6bw8147dTFHfjesFH_ytARlKw5bmfLL4,8493 +jedi/third_party/typeshed/third_party/2and3/jinja2/exceptions.pyi,sha256=P52TnpizVoU2vAeayt4MZlTYMnhJYiMrWxiLMWP1LeQ,1050 +jedi/third_party/typeshed/third_party/2and3/jinja2/ext.pyi,sha256=w5pYOcJ4h9vgSon1YceCeXsGDWjvUQjGSdCOhoScYXQ,1684 +jedi/third_party/typeshed/third_party/2and3/jinja2/filters.pyi,sha256=OxR24skT1uz7n2J-h3voluptB-l21pQ9KTx0CQnGsVU,2425 +jedi/third_party/typeshed/third_party/2and3/jinja2/lexer.pyi,sha256=evBj10x_89jlfuVacpADTOynp2XF9n8d2vQf5lZg-pE,2764 +jedi/third_party/typeshed/third_party/2and3/jinja2/loaders.pyi,sha256=EbGBtL810XARseWeDa6srKOlNUiUesvlJllf8Li-dw8,2923 +jedi/third_party/typeshed/third_party/2and3/jinja2/meta.pyi,sha256=_R2ui6gjeNSl5x59dwZjhL1WxTG0oKqXSCrgLx-2L24,339 +jedi/third_party/typeshed/third_party/2and3/jinja2/nodes.pyi,sha256=-86WsO5Cbws9IqXBSZidnIVlKesGmV2VFdhIUMrK8Vs,5271 +jedi/third_party/typeshed/third_party/2and3/jinja2/optimizer.pyi,sha256=1bIneJHqfKubBDxSn7sWv2-tGGZ06G6c4zr9Lxbd4X4,661 +jedi/third_party/typeshed/third_party/2and3/jinja2/parser.pyi,sha256=G6S7SzeUUTs_AGL916QYuaEKRhGyrpCekEVt0JOoRRk,2576 +jedi/third_party/typeshed/third_party/2and3/jinja2/runtime.pyi,sha256=8uryZzgqtGakyVDyA7OGzI9uqvSobgIoGrsj5MWmwb4,3463 +jedi/third_party/typeshed/third_party/2and3/jinja2/sandbox.pyi,sha256=zhmpDnPs50ugSYwrdg7V9r3OCbM839NHfBHILwfse-M,1146 +jedi/third_party/typeshed/third_party/2and3/jinja2/tests.pyi,sha256=dixzMe9BiHnvbL8KyR-KY9LLX-VD1-pgbWXC6Mwc4LA,561 +jedi/third_party/typeshed/third_party/2and3/jinja2/utils.pyi,sha256=oCtfBv7W9JqYQMU7nukSm8kVK1wy38U7db3oFsnXaUI,2957 +jedi/third_party/typeshed/third_party/2and3/jinja2/visitor.pyi,sha256=yeJ6g3RC5bitP3tG-btWLOZn954E_e9RxKPtgsiwcVc,306 +jedi/third_party/typeshed/third_party/2and3/markdown/__init__.pyi,sha256=31jve2aAqalclv4Hgat04sP28Oz06Nh8qwJVkC4dB84,146 +jedi/third_party/typeshed/third_party/2and3/markdown/__meta__.pyi,sha256=lugjndxerVWkOhrd9Mo0zDlN4utrCKIse4vNsqQFvxU,46 +jedi/third_party/typeshed/third_party/2and3/markdown/blockparser.pyi,sha256=OYDwhQquCvSSpAxoXCZ0z5kABsWuvCog4LqG3HEePE4,463 +jedi/third_party/typeshed/third_party/2and3/markdown/blockprocessors.pyi,sha256=Hd8EgV71NYEe-nx1m2WHIoQ4p7dXJbzYRiA5vXka1O4,1490 +jedi/third_party/typeshed/third_party/2and3/markdown/core.pyi,sha256=K4hpNVt2gTOP72HB8YSv-xf6uHu_qsUTuBWKnAb9D_0,2555 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/__init__.pyi,sha256=qaZiIY5BUSCBliWaMIWdX4v-ROFISLWzp6juNrXQi08,549 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/abbr.pyi,sha256=NWvh4SRoU2bI0X7gdc0Gc5FNu3y51gIkRyDNfU93ozI,428 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/admonition.pyi,sha256=tX58KZyPP03stonTmJnnvgR8J-QL8MT6i6ILHtFZ_8U,385 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/attr_list.pyi,sha256=cQaSO-jiaqUGZN1BZoISNSAvLM9m9Y8W2-DJ5Qi9ytA,498 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/codehilite.pyi,sha256=HRl9kJHv2nleEi84BdCR9cIZ1w4yiOauvFKdmsXgieg,979 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/def_list.pyi,sha256=-aeDlJbL4JICSTuizcoBkYIHmkEfb6gpkGQEWpMc0r4,360 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/extra.pyi,sha256=_cNunwRxZSkv3OT-m9oWf76bVVMcJiTkosNCFtH3cz0,197 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/fenced_code.pyi,sha256=wLqffeWN4EVg1IpvnMeHW9CTfWJ7WC3Vwz8fi-_XrBE,428 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/footnotes.pyi,sha256=B219DIvS8G0RbJu_ThwOD0qxXME5Ble7k64ak8dfwak,1751 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/legacy_attrs.pyi,sha256=UE5KQPkirTdRMr90C4WqPUpKKHOALOQpT4HkHPrPxNs,300 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/legacy_em.pyi,sha256=S5wb-FfHSfIq9o5Uc1ZeQm3BwfR7mjRX8g2km5EV90U,306 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/md_in_html.pyi,sha256=BFh_ogT03pnKfpqUb0DY5LLw9T330erK3NzOBMOkI1o,260 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/meta.pyi,sha256=3eRiqW2npoe0iuhFt5BgFVGjhlSSKc4vYOkDjYltaJ8,361 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/nl2br.pyi,sha256=mjR-zAlnbfZkttPcKAihZt-8vj9VcsOvADl9LzTgW2E,150 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/sane_lists.pyi,sha256=hqqMgizBkpFc4msXW6IQjsGDp3WFwfeTvRHxSMuKk5E,392 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/smarty.pyi,sha256=RsAPMqFAK3Qfz51st2J6SdY7s0yDJXQMnl2HbRtHDJc,1080 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/tables.pyi,sha256=Eg96sG_ww2G6kt6N4jlSwVA-LE834KZGp4-EgDacB9k,414 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/toc.pyi,sha256=fcqE9YDibJzb8P1Ox7WWczXi_34dCvpcPwXCLkrjagE,1178 +jedi/third_party/typeshed/third_party/2and3/markdown/extensions/wikilinks.pyi,sha256=Q2FIa1iPu0XMRqJTb8wewdPoTXEnJXZiFuYL-Zm0T1o,404 +jedi/third_party/typeshed/third_party/2and3/markdown/inlinepatterns.pyi,sha256=W_AMUX0BcD5vv8OJRL9D2hJfAc0BzK1hs12kOgKKr08,3022 +jedi/third_party/typeshed/third_party/2and3/markdown/pep562.pyi,sha256=pLjxji-A4xeBVYjntboexEwwoSp-JO4CbTLziUGvYqc,276 +jedi/third_party/typeshed/third_party/2and3/markdown/postprocessors.pyi,sha256=BRcuzX2JjjkfSk-OxVyYiPl27LjR9ak-u9kp2xZRVe8,400 +jedi/third_party/typeshed/third_party/2and3/markdown/preprocessors.pyi,sha256=oo3D5AAExEaq9GWiXhVblF37c2cojo7hjTsfh3IiOVY,550 +jedi/third_party/typeshed/third_party/2and3/markdown/serializers.pyi,sha256=V3npjXzO5oR8Ov6K-4JZTa96ma5VQGrtKDRjdvRvCG4,91 +jedi/third_party/typeshed/third_party/2and3/markdown/treeprocessors.pyi,sha256=_DfzQcWXflM9vqyQYWmPb0DfH5b8PEFTBRhBrpUEsaQ,469 +jedi/third_party/typeshed/third_party/2and3/markdown/util.pyi,sha256=4tEwF_04Z1zG_yu314kiwQCJUfnVO9AFLFoDUAU04a4,1584 +jedi/third_party/typeshed/third_party/2and3/markupsafe/__init__.pyi,sha256=RTRaycAzCzRQGvRT-ejSLNdvwg5kQsXB5oHcBPHypjo,2861 +jedi/third_party/typeshed/third_party/2and3/markupsafe/_compat.pyi,sha256=xN1vhSTmy2fXTE5pUrL_NRkYoOLH3lBY6YDf7ZR9czM,435 +jedi/third_party/typeshed/third_party/2and3/markupsafe/_constants.pyi,sha256=Kk2R4mLBd9wpcFgKA0k4u87FmXD-v1U6ODEduwhFkVE,62 +jedi/third_party/typeshed/third_party/2and3/markupsafe/_native.pyi,sha256=reb7nc_c28JTwKoUYlgirotBiOiESx1M1bpLZtXt6DQ,242 +jedi/third_party/typeshed/third_party/2and3/markupsafe/_speedups.pyi,sha256=reb7nc_c28JTwKoUYlgirotBiOiESx1M1bpLZtXt6DQ,242 +jedi/third_party/typeshed/third_party/2and3/maxminddb/__init__.pyi,sha256=VeOvFZHkaU9NmEfD-ZxOq735CGwfJmnMkDRUmkYLDW8,177 +jedi/third_party/typeshed/third_party/2and3/maxminddb/compat.pyi,sha256=lE11t1KuE4nA0ph6Bw9dWfhNE4l45r-4fCxOVRy4AfQ,194 +jedi/third_party/typeshed/third_party/2and3/maxminddb/const.pyi,sha256=c1LWH0XYtbcs9T0mJz5lyj6ZAVBwU2zZJdRk7Dd4Fc0,130 +jedi/third_party/typeshed/third_party/2and3/maxminddb/decoder.pyi,sha256=lZIOBE8OKJGu1fMmiH_n0b46qbmDxnXNsCq2Ed9oxzA,215 +jedi/third_party/typeshed/third_party/2and3/maxminddb/errors.pyi,sha256=ifm1u-yPLyPC5KPPF7F5vswnJ2X-TeGc5EgYXvWvn30,46 +jedi/third_party/typeshed/third_party/2and3/maxminddb/extension.pyi,sha256=k7yLS2l266OjFQfh4zmzOwO-46LpvnT3KM9FnvrZybU,1122 +jedi/third_party/typeshed/third_party/2and3/maxminddb/reader.pyi,sha256=X414tjdlsmCmS-fFfxFTqHn5-E8ifpb9_ckjzM-g9iM,1269 +jedi/third_party/typeshed/third_party/2and3/mock.pyi,sha256=Al3tA7zKNLXxUAeGa1e1ndsoaQtSm8ouPYT7_zifejI,14819 +jedi/third_party/typeshed/third_party/2and3/mypy_extensions.pyi,sha256=w1Erg7Whcf03pE8mI0T5qLwLPoaHncPRRJMrcgAomQs,2198 +jedi/third_party/typeshed/third_party/2and3/nmap/__init__.pyi,sha256=x_aIpzoj5vmzZKXKDrwrKEN5CWrg7XCDZ1-xOdfGOj0,20 +jedi/third_party/typeshed/third_party/2and3/nmap/nmap.pyi,sha256=hUW8b9yuNGhk4-XFq89rFhnzIghRIVN4wmYUiFwGjYw,3953 +jedi/third_party/typeshed/third_party/2and3/paramiko/__init__.pyi,sha256=0oM8LtFIAnlG-laK6WjyklSWeCGB9wdSzK7gVUbRlsg,1919 +jedi/third_party/typeshed/third_party/2and3/paramiko/_version.pyi,sha256=EeKXNTXM6-EkF3h5RGFiff1iafaFyZYoMscJOfcYuHc,65 +jedi/third_party/typeshed/third_party/2and3/paramiko/_winapi.pyi,sha256=yFTkwFHb32lUW4WD6Eq8onLhRGGCenZZMOxAdvUvYIo,2649 +jedi/third_party/typeshed/third_party/2and3/paramiko/agent.pyi,sha256=VmUg7we1YYno6H4zHgjCoPY7pTO3zzFHknFMC8oYwEc,2057 +jedi/third_party/typeshed/third_party/2and3/paramiko/auth_handler.pyi,sha256=qbisXIS9P2lmxsw2cSVxh7y6WIMLsD9YRbBMdPOtpjA,1934 +jedi/third_party/typeshed/third_party/2and3/paramiko/ber.pyi,sha256=4OoK5kdfYx8MVMfmbZLxMVuu5xLxaPJe8uTnu0n8_Bc,611 +jedi/third_party/typeshed/third_party/2and3/paramiko/buffered_pipe.pyi,sha256=mvCBsHuL3EIsyewdEIiwUZNYpn7AaSBxcqTH_md14lE,520 +jedi/third_party/typeshed/third_party/2and3/paramiko/channel.pyi,sha256=2smxhSShd9xXKGqHb5MqB6Bo23CpUJIi1dQwHBiOIag,3796 +jedi/third_party/typeshed/third_party/2and3/paramiko/client.pyi,sha256=-zCUwFWZIcgSSwVKiEnzC8jsuiPC_DGdJUNaw1kXQ-s,2932 +jedi/third_party/typeshed/third_party/2and3/paramiko/common.pyi,sha256=MfByis2G6kU5-y4FjhhRTQD5DQaP5OOFrc-ciC9JiKE,3167 +jedi/third_party/typeshed/third_party/2and3/paramiko/compress.pyi,sha256=_nA_oaAuza2Oxg0glDo-ov6uYbAIYlKcrkY5spOfqAk,296 +jedi/third_party/typeshed/third_party/2and3/paramiko/config.pyi,sha256=uxhOYKOIPM7jGe8ZsKKh9f_yNtKEdLdxfFC2mV9nwSc,1170 +jedi/third_party/typeshed/third_party/2and3/paramiko/dsskey.pyi,sha256=RdWJ9CuhBviY1c7FChwLDe_ZTOzLukxqaC1K5A6AmpM,1242 +jedi/third_party/typeshed/third_party/2and3/paramiko/ecdsakey.pyi,sha256=q8KVhmbEGp9_YW_9AYcEx_nKshajW_qzWl8QcKSTv5I,2338 +jedi/third_party/typeshed/third_party/2and3/paramiko/ed25519key.pyi,sha256=4aW1Q8tt2U2QIY5uVuSuAczVFBLkP8PR3_AgniR1tlU,703 +jedi/third_party/typeshed/third_party/2and3/paramiko/file.pyi,sha256=Lu1hmKapmlk3_jd1FcIPbrnz89WVlTNCwY46D4TFbfE,1342 +jedi/third_party/typeshed/third_party/2and3/paramiko/hostkeys.pyi,sha256=S1zc4QbgA7dGlEtcixKZ8WrT2Ok861yy-zU8DLtvZpc,1942 +jedi/third_party/typeshed/third_party/2and3/paramiko/kex_curve25519.pyi,sha256=naqQ5GRL0pVcTXD-_XQX1yXd2EpEk0Uu9vNRTfo1Xws,771 +jedi/third_party/typeshed/third_party/2and3/paramiko/kex_ecdh_nist.pyi,sha256=5n2l13AS1p4jz4LbYhSZiJ1Q5xYt4VYIUYLLvJ61ydk,1112 +jedi/third_party/typeshed/third_party/2and3/paramiko/kex_gex.pyi,sha256=NuQIADC7WgKxmDy2NCSzx4cTCDALv9x5Nlp_YxZ900Y,1023 +jedi/third_party/typeshed/third_party/2and3/paramiko/kex_group1.pyi,sha256=h9Vj1_2oQa6qTdB60kAF7MrpScKW9522kG3FL4SUmvk,679 +jedi/third_party/typeshed/third_party/2and3/paramiko/kex_group14.pyi,sha256=JyCccRukC_ojKvgpoVaTxH7dacQN5ICaJrRU6DX2_ZQ,453 +jedi/third_party/typeshed/third_party/2and3/paramiko/kex_group16.pyi,sha256=8unnknuU2cuPNUKhFO13YzhPsaOXewViTCsW8f9cw3Q,359 +jedi/third_party/typeshed/third_party/2and3/paramiko/kex_gss.pyi,sha256=7v4a1mbwTtuKVNmkU91yWlU7e-8VGVaq96xwsvB3iIw,1539 +jedi/third_party/typeshed/third_party/2and3/paramiko/message.pyi,sha256=6MGhnlQZ53mxrYlQ5JjloSks1tlqUJDAzc7RZeZ9E8M,1496 +jedi/third_party/typeshed/third_party/2and3/paramiko/packet.pyi,sha256=0X8hvx7r-I2siTS7L5IJVzf0GeAmX3BOHXcOxxJuOtc,2187 +jedi/third_party/typeshed/third_party/2and3/paramiko/pipe.pyi,sha256=qnF6412bon86OfW4FrlTJjey2atC9u3eDLQKqH8QDZ0,951 +jedi/third_party/typeshed/third_party/2and3/paramiko/pkey.pyi,sha256=ZtYonbeM4kI4Xu6_eM0wU7aLzHFsEX6M3sG96x7b_2M,1800 +jedi/third_party/typeshed/third_party/2and3/paramiko/primes.pyi,sha256=e2J5npj8PeYfnQq2pW1RLCgMofmSITjDVWH0Z2lmv0Y,308 +jedi/third_party/typeshed/third_party/2and3/paramiko/proxy.pyi,sha256=4VFZ2iL-P85RBWXliik--vmtlAUsngl4CT2giLlxuHg,504 +jedi/third_party/typeshed/third_party/2and3/paramiko/py3compat.pyi,sha256=KWCrIqjmlN5keG1E5Ns6jDa46z8yMym8hJpv-1Deuuc,1053 +jedi/third_party/typeshed/third_party/2and3/paramiko/rsakey.pyi,sha256=uHpfuSfDNTe7MsIdb9H6x5fKd1vP0SAKIcdOerxOTFc,1347 +jedi/third_party/typeshed/third_party/2and3/paramiko/server.pyi,sha256=lyv5SoRjT2tkm3AzCFG8CA9D1OtJKSdCCiQ5fNxfNi0,3062 +jedi/third_party/typeshed/third_party/2and3/paramiko/sftp.pyi,sha256=I81h3XktYUmZyfhBMsJJwWFYOxTiA60IAohoy_ekfZc,1065 +jedi/third_party/typeshed/third_party/2and3/paramiko/sftp_attr.pyi,sha256=7kLm3Cjr6Pm40VuoEzfV_u6zA_Ua2Pknu7-D5FxTpaw,667 +jedi/third_party/typeshed/third_party/2and3/paramiko/sftp_client.pyi,sha256=lgZedQtTqoeS7zSEO2r2Byo6mTO-DJxXgDxmQtRJva8,3133 +jedi/third_party/typeshed/third_party/2and3/paramiko/sftp_file.pyi,sha256=41HlM6ayMJaExhuvTI599gyKLtJhWOgxNTpIcXgkYlo,1356 +jedi/third_party/typeshed/third_party/2and3/paramiko/sftp_handle.pyi,sha256=OnJmN4k7typ9GyVtWukw4CuBvDIDvpWCdBQ4PxGPqJQ,531 +jedi/third_party/typeshed/third_party/2and3/paramiko/sftp_server.pyi,sha256=4aNH4sKvRI-49RNeOzFdSJAnvBnrgy0slFWrQ0AOPsk,1094 +jedi/third_party/typeshed/third_party/2and3/paramiko/sftp_si.pyi,sha256=RICAJB149dHYRnGFNgSwzZapr2Opr9-Eif4UCP1w7OU,1193 +jedi/third_party/typeshed/third_party/2and3/paramiko/ssh_exception.pyi,sha256=Nf27JN0Bt6JPpmMNEBC_LejuvbCi_P-ul9PYDqWd1t8,1454 +jedi/third_party/typeshed/third_party/2and3/paramiko/ssh_gss.pyi,sha256=WmUhwBWyvaof_6oP0ROFfrTo9OWccXMQ8_Q6USJq3bI,2774 +jedi/third_party/typeshed/third_party/2and3/paramiko/transport.pyi,sha256=kfJrcgrIs3Qu4KC7rd4EOcWMtz7ikEMeVqpgbvdMrVA,7945 +jedi/third_party/typeshed/third_party/2and3/paramiko/util.pyi,sha256=dxK3QYafkuBXobGkL0wBrZ3mJPz7ZwU-AFy2-1QaViI,1880 +jedi/third_party/typeshed/third_party/2and3/paramiko/win_pageant.pyi,sha256=X8oLeKZslmregVnQLAwaLDz8q95pCqVJ2ahSz3wVOI4,349 +jedi/third_party/typeshed/third_party/2and3/polib.pyi,sha256=kPN8NED1QJMdtkOwNC1Jh03d_i3iM8iaL7_vLaXoYao,5764 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vim/__init__.pyi,sha256=wddK6G7TpjbfWY1cCqoYTDgoLmqoUWTHC5cmy8he_LI,2197 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vim/event.pyi,sha256=GIxugpgYOODuc5rSewssVmYlRAZ8qmkC1C38rbD4yhA,395 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vim/fault.pyi,sha256=LYt9WpvRZNEwWlR55k6l7CxSFbS8k_FLJLjU-TGoP8U,195 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vim/option.pyi,sha256=YCaLxJmOtTldHIlo984EI6d9ykGsFbelNGuTHL70Ar8,204 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vim/view.pyi,sha256=hLDI0bDPEkPnxpLliSBlcmYgond-3RCxWc1_3NU-5Bc,586 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vmodl/__init__.pyi,sha256=DuLSUTy9Szmd-_pUmuOcSXoaWrnCPCH3HU9xUtW6AVo,74 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vmodl/fault.pyi,sha256=l8VLbRpKzACCz_wqwXLJkyxLwYRS0XiUl9JmKy2Vixw,116 +jedi/third_party/typeshed/third_party/2and3/pyVmomi/vmodl/query.pyi,sha256=h0HhTZVJIG0t0yvxp3bgwjVJEOJDyO1iDmPPShwj5C8,1430 +jedi/third_party/typeshed/third_party/2and3/pycurl.pyi,sha256=PkqfXT-xgGm4Sq1bKyxtX7Fk1kGI4Wx4mt2DFrqObRM,13755 +jedi/third_party/typeshed/third_party/2and3/pymysql/__init__.pyi,sha256=uughCRiT4wuIMgn5-CDOZwZWaFcJg2ZzWSqEVK6AU6k,1625 +jedi/third_party/typeshed/third_party/2and3/pymysql/charset.pyi,sha256=8pQ9uIfBVCKgQE42rgAV8Oc1RAUiR7c8k2z4w64zVso,327 +jedi/third_party/typeshed/third_party/2and3/pymysql/connections.pyi,sha256=Eno05u1fJ2WBME7vc-WAzP0ztvE4Khh9JbZ1VqI0Ci4,5627 +jedi/third_party/typeshed/third_party/2and3/pymysql/constants/CLIENT.pyi,sha256=vnIE4D-CCZ2-TwRYvficDcBx-luonNhHmsuxq8M0V6U,308 +jedi/third_party/typeshed/third_party/2and3/pymysql/constants/COMMAND.pyi,sha256=SAol0UV6ttKwz1i9DUn_DydO1HByeSodgDC3Qyv1BTA,407 +jedi/third_party/typeshed/third_party/2and3/pymysql/constants/ER.pyi,sha256=0O0FmmEhzeeEUmlkmSNluHuZVhhlTOIEHq1Yc-hXLFY,11280 +jedi/third_party/typeshed/third_party/2and3/pymysql/constants/FIELD_TYPE.pyi,sha256=112f_ZZ3GXSiGv5q3fY04FLq-NMWWh8zDSsneoGKfzE,354 +jedi/third_party/typeshed/third_party/2and3/pymysql/constants/FLAG.pyi,sha256=LJ19iNTKdV_tqShn5GyJbTc3JtGr7IesoCQy058KPx0,226 +jedi/third_party/typeshed/third_party/2and3/pymysql/constants/SERVER_STATUS.pyi,sha256=uoVJx08QJpMZzBiyZXTG1Hr7zcSahGx2zR89eOtYxec,331 +jedi/third_party/typeshed/third_party/2and3/pymysql/constants/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/pymysql/converters.pyi,sha256=JNCfG3LgOn6QUtAQm7Dkf5j5MAKasP9tOLDWGzD30jc,1331 +jedi/third_party/typeshed/third_party/2and3/pymysql/cursors.pyi,sha256=V0DYB2ILjVUjOc79rveMPALidnQ3ZlfRRiv_ZS608Hw,2180 +jedi/third_party/typeshed/third_party/2and3/pymysql/err.pyi,sha256=vz8muipV5ryF9NuS_6aya2mWwgAJ9JzdhAN-NvyhyBk,606 +jedi/third_party/typeshed/third_party/2and3/pymysql/times.pyi,sha256=N3iYLkF3MEtwRa8muKj4ViRXS7ON8fcW5XANu2VMxXo,170 +jedi/third_party/typeshed/third_party/2and3/pymysql/util.pyi,sha256=fKG9sTGCjyLG5L-n5K8STqna8XVfKzQ2a-X46ozbk20,66 +jedi/third_party/typeshed/third_party/2and3/pynamodb/__init__.pyi,sha256=qp8cF6TlhBd5820jKiBRcrR-qumnONUUlUKCzf4rib0,17 +jedi/third_party/typeshed/third_party/2and3/pynamodb/attributes.pyi,sha256=-GsbmMFuySBUZVphL5s770L_BinXheI-_8jEZWFcaa0,4342 +jedi/third_party/typeshed/third_party/2and3/pynamodb/connection/__init__.pyi,sha256=tIcusRqjD5cbw9gURA6V6gJS3kotd6qBxua-WeuUSfg,135 +jedi/third_party/typeshed/third_party/2and3/pynamodb/connection/base.pyi,sha256=lRRsdiy4q2wxWEKFd9f70MNqsUvGiaz3HbEYW366KXA,6089 +jedi/third_party/typeshed/third_party/2and3/pynamodb/connection/table.pyi,sha256=ybdRMvW7ihut6ROMqnP8Di_1janFYH82iN2BAO4Rreg,3825 +jedi/third_party/typeshed/third_party/2and3/pynamodb/connection/util.pyi,sha256=VelfJ8xvQieSdjx4hkp5g0W73tyArG2qobRxY2xccWs,67 +jedi/third_party/typeshed/third_party/2and3/pynamodb/constants.pyi,sha256=2DxSY2CwYrKx_uT0QKHorzmekcf6bcKWxQ5lZp6PRco,3038 +jedi/third_party/typeshed/third_party/2and3/pynamodb/exceptions.pyi,sha256=oN2x-x1qyp5WVRK-jF9rVjzQIHiUjNO5RAwfRTQSRdA,887 +jedi/third_party/typeshed/third_party/2and3/pynamodb/indexes.pyi,sha256=a5MVXShQkb2UUh5hJiX1S_UaLJ3Rzqp2IKB9ZfuU2SQ,1052 +jedi/third_party/typeshed/third_party/2and3/pynamodb/models.pyi,sha256=zg1iyUvCjODMELDKSPXncRXAZfyr-TePSzI61qfjUeo,5702 +jedi/third_party/typeshed/third_party/2and3/pynamodb/settings.pyi,sha256=nfytWdsaKGyoTVPsIi0B-15yecRvNQ61Y3L_sT0qpvo,145 +jedi/third_party/typeshed/third_party/2and3/pynamodb/throttle.pyi,sha256=Xnxx2caEi3AgLinoHSCVZUkuEPV5XxwMw83tpUaUvRk,472 +jedi/third_party/typeshed/third_party/2and3/pynamodb/types.pyi,sha256=BCI-zF5K_mOWPzGVixMvVXqISqSxWokGjR0aVosyvK0,57 +jedi/third_party/typeshed/third_party/2and3/pyre_extensions.pyi,sha256=CJ0jUa1yruR4a_cVqcFRpORBpYGea9C5vQ5lwGfWUZI,267 +jedi/third_party/typeshed/third_party/2and3/pytz/__init__.pyi,sha256=-LDivLI7EkEywpVQfVyMFuVNEDZFbod-zNxGNlQO_A8,1892 +jedi/third_party/typeshed/third_party/2and3/redis/__init__.pyi,sha256=pNlwX9y10ONwvlK1yzT9MMA6hQx06nW8u1qHrVpMmEE,829 +jedi/third_party/typeshed/third_party/2and3/redis/client.pyi,sha256=hKMjp_LRcqsk8xf2tv9HRdFVqqqJ46QH1MHbRgAReIE,26034 +jedi/third_party/typeshed/third_party/2and3/redis/connection.pyi,sha256=aMlTz2YmsHb6igv5yajn4gJswPs9frFa_grs2eEqCLc,5595 +jedi/third_party/typeshed/third_party/2and3/redis/exceptions.pyi,sha256=d2fmJa9u5GrrAaST8JgdfpFsBxKupvKLb2iH9VgP7Wg,569 +jedi/third_party/typeshed/third_party/2and3/redis/utils.pyi,sha256=a5cyDCASB0E1ZuBKK33dZI7ZBcr2rISkiWU5alhjJbA,136 +jedi/third_party/typeshed/third_party/2and3/requests/__init__.pyi,sha256=LIORHwdjl-Ogkvvd5Fs_GZS3k_LcEOd9qglRpTQnYDs,928 +jedi/third_party/typeshed/third_party/2and3/requests/adapters.pyi,sha256=OfzqJV64CBHmzwroW2Tm14e5kZyCu_L6NJWSUhBgKLI,2925 +jedi/third_party/typeshed/third_party/2and3/requests/api.pyi,sha256=Gcw_jo5GO0jbLDMvukc10WFuXCngBEjyTbTTNgvgeMY,1216 +jedi/third_party/typeshed/third_party/2and3/requests/auth.pyi,sha256=q63njtnTjj3AICWDacBKsIS7DjLikQsJwRd7CtWa26o,1148 +jedi/third_party/typeshed/third_party/2and3/requests/compat.pyi,sha256=d3SHDa-UnNy17Mv0aio7kQEf6qsRzEpgA31-MwWuBIo,58 +jedi/third_party/typeshed/third_party/2and3/requests/cookies.pyi,sha256=wn0Y-_dNxjgTjY9WfRuJplkBLY41YyhUS4cDAiFGGo8,2033 +jedi/third_party/typeshed/third_party/2and3/requests/exceptions.pyi,sha256=RxHFR2hELgvFoBEa0AKeYaWNmjDyrWJrvJ6RKAr24UI,1259 +jedi/third_party/typeshed/third_party/2and3/requests/hooks.pyi,sha256=daf3Tp5DknV7lyWtGe5n0jV7U0I_6HwoMpOnlgCdYv0,117 +jedi/third_party/typeshed/third_party/2and3/requests/models.pyi,sha256=ynI5o3SRKsMTkXyxba2NCAVdTtGgC1hSqQj-4QXMLSs,4578 +jedi/third_party/typeshed/third_party/2and3/requests/packages/__init__.pyi,sha256=cnNTJT8mNb7zJ8vT6jKdpiWxIYAZMw3Qz6frelGgzbs,158 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/__init__.pyi,sha256=noXIj8HyCJNtp4CXGIfsy6-yAi7pT1P4uTn-_vYdGPo,825 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/_collections.pyi,sha256=aHphg2k55d7e0tqFKmSyIvsa2vRgJNA3rr_GvV28BF8,1535 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/connection.pyi,sha256=gW4ugIttzymKZSoRvGOSD65mT69TANr35_NBfNIn8mk,1907 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/connectionpool.pyi,sha256=CQdrc51bqLxZG_Sd_u23SPcIxffQtggfp1sS8FJLiog,3170 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/contrib/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/exceptions.pyi,sha256=TxvMswYXJdgim9frBpmQbC5q_Cu8SM37vW2ZFrfvH20,1413 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/fields.pyi,sha256=ol7EAXCmRCqnANBpCeua5OHNjWGz3aCVf_2j-eIZlgc,442 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/filepost.pyi,sha256=VdQ66F4SjusF9qTkUP8U0yUbrRqIJv8DWfTISgJ7N3g,244 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/packages/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/packages/ssl_match_hostname/__init__.pyi,sha256=8-WNQ8HsF7JWBeN1n0qoPU84HsAouCrHj4R5qrGKrVQ,88 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/packages/ssl_match_hostname/_implementation.pyi,sha256=jAdLMMvdxdszZCIdVGxRZw7i5waIBqb4RqwketJqMr4,81 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/poolmanager.pyi,sha256=66XnVPM1snW_kgKDX2_21PwyN-As7xtgAiCEHXf6U14,1309 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/request.pyi,sha256=chC2ml6eAcw5E2ILdPySz_aZ9OQMeD5nx0hlmYVfkz8,534 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/response.pyi,sha256=Qpkf8CpEW5s2pKMxwAVfgz2zQ4miW7kyd-FiOeHyvG8,1788 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/__init__.pyi,sha256=xacNymxEsCZp5wlKo15L8WSYXzhruHcRtGMqamjArZM,615 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/connection.pyi,sha256=XaFYtI2YyZGnOVivtXsLW6jMSb6FKpfIh3aIQInJLR0,188 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/request.pyi,sha256=hbniWmj2f5WG5m6qurjL54K3VzMhAffiSMa-rr1CO0s,227 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/response.pyi,sha256=Q0CWgpQG5zMPxwc0LOr5eQOEfmOqglfpydPfBVxiStc,27 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/retry.pyi,sha256=DWGmbAvdoZc9gL9-7N-wU4A-SmtfWYpKmSRoZFno-kc,1161 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/ssl_.pyi,sha256=VY_E6OYlixXdjHOT_FYYFjaECs7n8BxzU1O9rAvp1hI,691 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/timeout.pyi,sha256=EojIS91DNeDBfY9TONTAuJkoR6LmHGeX3IYNmLZdvWY,499 +jedi/third_party/typeshed/third_party/2and3/requests/packages/urllib3/util/url.pyi,sha256=iXePrO9HE5OJVXeWx0nu4PKX8ZX3ScJt1KgPrYpUtrA,492 +jedi/third_party/typeshed/third_party/2and3/requests/sessions.pyi,sha256=qpxhe7nKCvlYZbWU_HcgynucxH5vvj7D31tEIkNq6R8,4727 +jedi/third_party/typeshed/third_party/2and3/requests/status_codes.pyi,sha256=miO3NECO-3yQmK56zg67HzZSbx1eER1xmKABjoDMuXs,35 +jedi/third_party/typeshed/third_party/2and3/requests/structures.pyi,sha256=7Dr21XygxBfDVhOr83FfWEiJNt_W6uNzOmyLfermE_o,968 +jedi/third_party/typeshed/third_party/2and3/requests/utils.pyi,sha256=cYgq8lcqt9s25H2SfQ_TuTvZfGFMlpJEH6yW43yX_IE,1989 +jedi/third_party/typeshed/third_party/2and3/retry/__init__.pyi,sha256=oURuYy9CoiIEbeBziwmNwAexn1hXNQlhL1_lH15UZZw,32 +jedi/third_party/typeshed/third_party/2and3/retry/api.pyi,sha256=fnNyTJFt_8RiZ5QT87GTrUPQvH8Lqrwm7WXT74s51-g,935 +jedi/third_party/typeshed/third_party/2and3/simplejson/__init__.pyi,sha256=r6sZ4B_hGIz2y83FiFs5WvQweXydgb2GfykOzF0Gd3g,538 +jedi/third_party/typeshed/third_party/2and3/simplejson/decoder.pyi,sha256=ijcE8aSOnNvjIOQ0s3eiKjq7GYcIOGe9_lESZPAmweM,229 +jedi/third_party/typeshed/third_party/2and3/simplejson/encoder.pyi,sha256=Ju-aWi4B6WNQ3G-6n7VQw7idFNeKMhS9g_b7XcHSXLo,264 +jedi/third_party/typeshed/third_party/2and3/simplejson/scanner.pyi,sha256=_AUT1GJRNYopQ98J8rRXtig5nCunjXLoiCf5gN9dpXo,262 +jedi/third_party/typeshed/third_party/2and3/singledispatch.pyi,sha256=snB5T00WMlJzaeON3HT586VTC_rlHBZ5d4Q-Seaj1VM,624 +jedi/third_party/typeshed/third_party/2and3/slugify/__init__.pyi,sha256=oS6sazgR3xHmORcsqDRk6ZulbTzRF1UJaDw1V2i9428,46 +jedi/third_party/typeshed/third_party/2and3/slugify/slugify.pyi,sha256=dRDus-8qkVm5OBBP31a-RdSahYcrn9HGYJcgTa_LPBI,561 +jedi/third_party/typeshed/third_party/2and3/slugify/special.pyi,sha256=Nw6LzOLeZREXSk-gK6A_pqAcurWVlbnE-TAzw6pkMwk,279 +jedi/third_party/typeshed/third_party/2and3/tabulate.pyi,sha256=JJ7kYLQU-3I7s68wXecrchQU6zKeYPHH7mb6yGdcsHE,1413 +jedi/third_party/typeshed/third_party/2and3/termcolor.pyi,sha256=bvMVbmC_0sADtLLJ1gTgz0qy7u6z0nxMmPyumxJurUM,350 +jedi/third_party/typeshed/third_party/2and3/toml.pyi,sha256=wqxUoqPomQnEzd6_LnLALaHqfgqKJH2O4wbLkCBA61M,697 +jedi/third_party/typeshed/third_party/2and3/typing_extensions.pyi,sha256=Yv2B_WZlMZ9ZF5LzirIeruXF56kHMW22DLFGtnx_3l8,3358 +jedi/third_party/typeshed/third_party/2and3/tzlocal/__init__.pyi,sha256=gZwxQC65kbnoMEOnhjkBKqvx2aJayhHsQ52R1s_Bl2E,104 +jedi/third_party/typeshed/third_party/2and3/ujson.pyi,sha256=YuEF8oRji3rvUknBhjJBa8bJqdZD2H07ZGDIUwyjSP0,938 +jedi/third_party/typeshed/third_party/2and3/werkzeug/__init__.pyi,sha256=Z4tIEffX6wFRXuakZjYqPUNOnfUbr7STWtzAAWlF11E,5307 +jedi/third_party/typeshed/third_party/2and3/werkzeug/_compat.pyi,sha256=XYwsJu2d8JZH41AXqSDlRaODFAG8hzsrIVrjlnHAaDE,1271 +jedi/third_party/typeshed/third_party/2and3/werkzeug/_internal.pyi,sha256=xff0Vbs8wNEN9bykzs7k9bgLdhDY-DiIUj1NbsY0ZOE,644 +jedi/third_party/typeshed/third_party/2and3/werkzeug/_reloader.pyi,sha256=QCQ4dGw3MM4rhwKmKntyLJ7JB-neMbnRuRrm2scpZgE,826 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/atom.pyi,sha256=LAPRofmeDhSpU418KTzoATp5cB2D9daWgcgP_A2YrDI,1136 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/cache.pyi,sha256=qzO8UcufWrYMdYREIG3H0rutk922G-3guC4zNJoWFlI,3375 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/fixers.pyi,sha256=mvPRIfbtHVRE98seLNNyHSonlGnl7uxSZC0s9TfSEa0,1650 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/iterio.pyi,sha256=2yRg8voINYQF23YfRkpGPXaaaX4jpupooKthKwfX4kU,1202 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/jsrouting.pyi,sha256=4s_tRK57Yu_n1O1Zk85Nty48UaBHu6AsgeHHxLTS0ko,325 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/limiter.pyi,sha256=qLbYIPMTBsgaYe74m3NerbQ1AkW_57LKwY0iW_zaIXU,192 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/lint.pyi,sha256=vnPiJs5jXekVZmugPlmkXITga2URy2gh6vngdxkn_Ls,32 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/profiler.pyi,sha256=HtjfNVB5ydRd_FfwMkAzJ05aHtFcN2Eecjf2nHQfJVM,315 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/securecookie.pyi,sha256=EUmDrvenhGHAJ9GUO20vYOlwNon9XoAVABBxREOqnBs,1212 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/sessions.pyi,sha256=7V-lJ3FrbkOIp0RK42KtlhntH3SinErVLH78sgu5Zes,2074 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/testtools.pyi,sha256=OOwXK52N2t35kMCMl0B4zBK-d85PcqGG_nOAbhsEAD8,188 +jedi/third_party/typeshed/third_party/2and3/werkzeug/contrib/wrappers.pyi,sha256=AdT0yvz1TaiY0mn6zlt_ET3DKsTywVVjkErC769c6L0,603 +jedi/third_party/typeshed/third_party/2and3/werkzeug/datastructures.pyi,sha256=nzcm32yD5-va5Toukm8sXENL4F6AUFxyAD3iAkF2OQg,15587 +jedi/third_party/typeshed/third_party/2and3/werkzeug/debug/__init__.pyi,sha256=Th_QGEjQE3niHm77xHcmtgRUFZxFd2IsIUhqNGymRdE,1384 +jedi/third_party/typeshed/third_party/2and3/werkzeug/debug/console.pyi,sha256=DBWA7QK0dluokniouOizZ1h5oAIcWi47N7i6UiSDRiU,1207 +jedi/third_party/typeshed/third_party/2and3/werkzeug/debug/repr.pyi,sha256=dqtVFNOfcjINAMzaJ-tHiW6SO37ZwVtVT1J7SS7yWh0,846 +jedi/third_party/typeshed/third_party/2and3/werkzeug/debug/tbtools.pyi,sha256=tbrAcENEbWQEOxxSrEKr_5GA3i-OaKn7K5QPA4Sk6TU,1687 +jedi/third_party/typeshed/third_party/2and3/werkzeug/exceptions.pyi,sha256=xvGHuWowxhPqNfh4VqLPjTsRubPoapE3-Jo_k7T3UMk,4942 +jedi/third_party/typeshed/third_party/2and3/werkzeug/filesystem.pyi,sha256=te3jyfyiDfsnFU0MEYw-wB5jBgWgw4VAbIpQzGAs9Vs,169 +jedi/third_party/typeshed/third_party/2and3/werkzeug/formparser.pyi,sha256=cr1vdd95MjSLCA0VyEutG3F4ezKQpuY54LSj3d03F7M,3702 +jedi/third_party/typeshed/third_party/2and3/werkzeug/http.pyi,sha256=gQCvsTijEmaeJXpTMvKOFt_o-XTAtNtfSggTeBqhSq4,5335 +jedi/third_party/typeshed/third_party/2and3/werkzeug/local.pyi,sha256=ZWdRsm--5mtPjtmAH_7-0JBQmvQUfAba_VbReVyAVIg,2315 +jedi/third_party/typeshed/third_party/2and3/werkzeug/middleware/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/2and3/werkzeug/middleware/dispatcher.pyi,sha256=CYiiHkHrQ9Qlc0JmtZvUBWUPxeUJDjaxzwBbriStxo8,451 +jedi/third_party/typeshed/third_party/2and3/werkzeug/middleware/http_proxy.pyi,sha256=5S5vt4U681ugR9lV5G6ZFkopoSw9tHCocHenpeSJAA4,652 +jedi/third_party/typeshed/third_party/2and3/werkzeug/middleware/lint.pyi,sha256=fKmZa4yVYgObsnOwlVaYh7jo9bQrVQu4BA15g2uzeLY,2389 +jedi/third_party/typeshed/third_party/2and3/werkzeug/middleware/profiler.pyi,sha256=_DWwg99Hb3nYORrYOngd_L6LGGV5rWjIC1EXuucQ6LA,569 +jedi/third_party/typeshed/third_party/2and3/werkzeug/middleware/proxy_fix.pyi,sha256=oJCpfNfoF8oKte5Ikmu4XBdYv_4pJqgFGOf86Iq47u8,712 +jedi/third_party/typeshed/third_party/2and3/werkzeug/middleware/shared_data.pyi,sha256=JVOyPz2zNzW2AlLdc0uAQQ10SQ3Ved-rLzv3cVdwVwA,1295 +jedi/third_party/typeshed/third_party/2and3/werkzeug/posixemulation.pyi,sha256=L5e2d7yb54c171JGU8g_r6G4qIDx4Av28BY12iiwT6w,199 +jedi/third_party/typeshed/third_party/2and3/werkzeug/routing.pyi,sha256=dY-lLGVr0rQ-p_y-8xP5papvCEuSBuaFkNeC8ADEN_M,6802 +jedi/third_party/typeshed/third_party/2and3/werkzeug/script.pyi,sha256=ifTEJ5yx1gam9_q9RnsxmiN3W4GRvegb2kRTfZ-z_0g,768 +jedi/third_party/typeshed/third_party/2and3/werkzeug/security.pyi,sha256=utyGUghv_u_Rsb4p48VUyXuDJuLo621MJZuafdW4Gd4,524 +jedi/third_party/typeshed/third_party/2and3/werkzeug/serving.pyi,sha256=gNa8EOCapHswvpWYh4qYN1783afRO18_26L5jYmGRaU,4057 +jedi/third_party/typeshed/third_party/2and3/werkzeug/test.pyi,sha256=ICgRxquqP1H5-8Tlk7UAn2DXcl7IvF2ZV_7LV93SZng,6138 +jedi/third_party/typeshed/third_party/2and3/werkzeug/testapp.pyi,sha256=O0O2-rhExeAFHWIryQcjWpQhGPQfkkU8SuV0uLbbpco,226 +jedi/third_party/typeshed/third_party/2and3/werkzeug/urls.pyi,sha256=CKx_xMwXyR1EXEx7DFrXZIr8OdrpyZ3m2htWiCWGvgQ,2898 +jedi/third_party/typeshed/third_party/2and3/werkzeug/useragents.pyi,sha256=4LzBjciKyh8zZcufKX8SjjvDAu_rrCsEdxr4-LwZbKY,431 +jedi/third_party/typeshed/third_party/2and3/werkzeug/utils.pyi,sha256=kPk0JhxbcvYk-AAWyo3s-8nhsUxm32PpEa5hjVTilHw,1962 +jedi/third_party/typeshed/third_party/2and3/werkzeug/wrappers.pyi,sha256=uWHxu1UBSC9WlMmZ-HbIk71sgRb0-euFZ5lY4xUh5g0,9115 +jedi/third_party/typeshed/third_party/2and3/werkzeug/wsgi.pyi,sha256=1TOv_UGcc16Znog_D-iNwXHnnlIwSfFk97mA_zcL2cY,3030 +jedi/third_party/typeshed/third_party/2and3/yaml/__init__.pyi,sha256=bElbXZu32G81wGc_WxNeXTnPxSStYTaJFRzrPjj8kxU,6559 +jedi/third_party/typeshed/third_party/2and3/yaml/composer.pyi,sha256=Vl74-4GCncmKS-eb06xYTS5KZtmmOl_4DlXceFg_lbY,496 +jedi/third_party/typeshed/third_party/2and3/yaml/constructor.pyi,sha256=K8dllpXyNJoRWMHkbGLYZJvnoNO33n3nv3VCdRWOR8c,3603 +jedi/third_party/typeshed/third_party/2and3/yaml/cyaml.pyi,sha256=1mixUYME26X3XC9R1onrnbH91b5S94KPpnVZRc0Z0CA,2292 +jedi/third_party/typeshed/third_party/2and3/yaml/dumper.pyi,sha256=Q3Byffe7aMWtkkC0wdCgWognnXnQch-sHziRFxQASdo,1548 +jedi/third_party/typeshed/third_party/2and3/yaml/emitter.pyi,sha256=TPp-ubnhpVa6Vp89RdhjjfIMbMoI35XSj5t5_pL5OSE,3802 +jedi/third_party/typeshed/third_party/2and3/yaml/error.pyi,sha256=kYhKOUoSrnEQLvgIvG_qmPKgMQ53ADqFHOPPDTlRTs4,535 +jedi/third_party/typeshed/third_party/2and3/yaml/events.pyi,sha256=IXJuqeHbEQtHZ4mD36QQ1-CAQaLbKua8JEZ-qh7jPKg,1662 +jedi/third_party/typeshed/third_party/2and3/yaml/loader.pyi,sha256=Q_smlF0bYLSXJ8O9TH93ErQmInWH0S8dqibxbXG6API,767 +jedi/third_party/typeshed/third_party/2and3/yaml/nodes.pyi,sha256=HXEU6EM5Z7O0WqO8hCQbRXW_BCGYAzK68xKTZC5yDJs,685 +jedi/third_party/typeshed/third_party/2and3/yaml/parser.pyi,sha256=0uGiWaCcBAoSQBsXUVRqYGAevPu8GLG5ANORAGS_snE,1664 +jedi/third_party/typeshed/third_party/2and3/yaml/reader.pyi,sha256=i91dNQcOijQHle6687O3xJHWAVy5Sq_k-ko79nolORo,832 +jedi/third_party/typeshed/third_party/2and3/yaml/representer.pyi,sha256=J7rrcGo3XvshSlUyHGbJXMprquSaD3oTHv51VwqWr3Y,2185 +jedi/third_party/typeshed/third_party/2and3/yaml/resolver.pyi,sha256=h32kNW_kLa4Ckxc8DxTHcsFvvgvYKcQQ7lFsV3Pt3s4,786 +jedi/third_party/typeshed/third_party/2and3/yaml/scanner.pyi,sha256=Ly12oPYyJVNrdODBM-F9dVO1EzK3hiiItMgay0hDQAE,3573 +jedi/third_party/typeshed/third_party/2and3/yaml/serializer.pyi,sha256=W27WLPu0g0Cm6vnExbuG3NGLdzUK8tuPM17SSEwY4HU,666 +jedi/third_party/typeshed/third_party/2and3/yaml/tokens.pyi,sha256=lRuHaCHbPMRTOm3WEhanBssRRnJHUiGh1mjNmE4CND8,1792 +jedi/third_party/typeshed/third_party/3/aiofiles/__init__.pyi,sha256=JV9ztGDgZgmwrcgLqJ_R6o8Fgf3cmJWE5MgYjP2BEE0,37 +jedi/third_party/typeshed/third_party/3/aiofiles/base.pyi,sha256=gL3zcEhpb3ZeAHKbouMayq0UIzTDrsE8oTwKU8Yhyq0,1490 +jedi/third_party/typeshed/third_party/3/aiofiles/os.pyi,sha256=G9ZSixzRK7_9tZ5sSjqhp-c7Uy5f8tfI_SEQUwYnScw,1040 +jedi/third_party/typeshed/third_party/3/aiofiles/threadpool/__init__.pyi,sha256=plwVo_uI0AiOap217i7Ef_Gh_TGyOOeJren5WnTIZds,2812 +jedi/third_party/typeshed/third_party/3/aiofiles/threadpool/binary.pyi,sha256=xmn9nxqyzM5XQCr5H8sx2xzW7YkqAOQ159cL1yljyIA,1619 +jedi/third_party/typeshed/third_party/3/aiofiles/threadpool/text.pyi,sha256=X4W8oOVHyTrz32gJ-W_-rYQIw9A92MUd1pp3okUG3xY,1416 +jedi/third_party/typeshed/third_party/3/contextvars.pyi,sha256=vR-ubUfhlQVgLJbJzuVOM5UTc-sFCVSEg2EzcqLMxzI,1405 +jedi/third_party/typeshed/third_party/3/dataclasses.pyi,sha256=KYjW7iSlZ42rrD1Z34CvtIjUSzOTm2lcLozDVSAOkSo,2737 +jedi/third_party/typeshed/third_party/3/docutils/__init__.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +jedi/third_party/typeshed/third_party/3/docutils/examples.pyi,sha256=3DheJVl7ojvJt4_YF7b1hRFNExi-vO9IWxtfojfi0uo,80 +jedi/third_party/typeshed/third_party/3/docutils/nodes.pyi,sha256=yxmAtZCJ_mCnQ6pH3u3qACugJ_Tpis4DrrkGGTyIcTk,203 +jedi/third_party/typeshed/third_party/3/docutils/parsers/__init__.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/__init__.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/nodes.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/roles.pyi,sha256=7P4PG1OuDMfSK1DnTQzcOtAeQVqUViHtNXrn3G8hjCo,418 +jedi/third_party/typeshed/third_party/3/docutils/parsers/rst/states.pyi,sha256=qb0d-B857-wAVpdJ7vT06HNQRkpaTDj1YXl3cOWzqVs,115 +jedi/third_party/typeshed/third_party/3/filelock/__init__.pyi,sha256=diWa8QPjqlZ7BhKzJxWLqfFHOgE3XYRWfe06UbvgWlU,1789 +jedi/third_party/typeshed/third_party/3/freezegun/__init__.pyi,sha256=xM4QVDAOY8pmP9Isl_PcX_TU5zDCY7rP0U5dY9pWQzE,44 +jedi/third_party/typeshed/third_party/3/freezegun/api.pyi,sha256=6Infm2WmoBzTquz0uTNPmAYJ9b8sE8cEHuWo2tMaPLY,2266 +jedi/third_party/typeshed/third_party/3/frozendict.pyi,sha256=E89ILrzNgYcRBsm_vnQLOvKuit1t2WhnQVdq60lLeuM,895 +jedi/third_party/typeshed/third_party/3/jwt/__init__.pyi,sha256=975_sUWTv0BaJ04jZpVTmd7WNFmu25f82KqqMReqI3k,1724 +jedi/third_party/typeshed/third_party/3/jwt/algorithms.pyi,sha256=Vp1NVzBfM8jz0VJGhop3UOnKXa1mmJSNI3u0iv_-N70,4299 +jedi/third_party/typeshed/third_party/3/jwt/contrib/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/py_ecdsa.pyi,sha256=43zFcZpLRNwzuol7DXZi7jGaQ0M6jDzfyTNeWsuwtRk,251 +jedi/third_party/typeshed/third_party/3/jwt/contrib/algorithms/pycrypto.pyi,sha256=w-Prvq6u6oOOCt6KPuTT1p86AW7cABIOj8iDa00gvyM,252 +jedi/third_party/typeshed/third_party/3/orjson.pyi,sha256=BKaBOXNbLXb2MHYpT8BwCh78j8HagLQq51uZY0Dq5nU,656 +jedi/third_party/typeshed/third_party/3/pkg_resources/__init__.pyi,sha256=h-i-Ph-0RjoeDFw5lowMgeyOK73tbkz_iaP28vyYUsU,12105 +jedi/third_party/typeshed/third_party/3/pkg_resources/py31compat.pyi,sha256=ouFLzqmJqfMmT4orlwN1utMnk2R5vHeXxDSk2FI-7I4,56 +jedi/third_party/typeshed/third_party/3/pyrfc3339/__init__.pyi,sha256=GdEm76Fq93-yKV0jkd5FMP5HR8CCpDV6-1R888e5sI0,79 +jedi/third_party/typeshed/third_party/3/pyrfc3339/generator.pyi,sha256=Y8yuzXtNLGik4y6l4TgvvU9ehZGNUK0Pe-MZVvddL4A,139 +jedi/third_party/typeshed/third_party/3/pyrfc3339/parser.pyi,sha256=E4529DNPk1MqcLAd-DgDjRo9onMQzi6rquEEIDZgZ_A,118 +jedi/third_party/typeshed/third_party/3/pyrfc3339/utils.pyi,sha256=wxuOjaaxzHH_PlMf8YxP4oVh5Oq_QayX3Kdfz9CHD_8,447 +jedi/third_party/typeshed/third_party/3/six/__init__.pyi,sha256=43jcx_IPU45XeUJ44qydi-un8-jRZFH4kya6OUlkuvw,4169 +jedi/third_party/typeshed/third_party/3/six/moves/BaseHTTPServer.pyi,sha256=vtBUJUabTrIVK97Cn0MhLUhHSlbmHB8QgQlWwadH-6w,26 +jedi/third_party/typeshed/third_party/3/six/moves/CGIHTTPServer.pyi,sha256=vtBUJUabTrIVK97Cn0MhLUhHSlbmHB8QgQlWwadH-6w,26 +jedi/third_party/typeshed/third_party/3/six/moves/SimpleHTTPServer.pyi,sha256=vtBUJUabTrIVK97Cn0MhLUhHSlbmHB8QgQlWwadH-6w,26 +jedi/third_party/typeshed/third_party/3/six/moves/__init__.pyi,sha256=Vsp6DhxVU4xIcJ8eDyruQNQz2a_ERZmnHjA2_5MOyTk,2363 +jedi/third_party/typeshed/third_party/3/six/moves/_dummy_thread.pyi,sha256=QcsaN0JxBr9ArwQnzhmN06G8dMTpqIuIbapJvWWr8IQ,28 +jedi/third_party/typeshed/third_party/3/six/moves/_thread.pyi,sha256=An3Es1KPMtE47GK-HKV_WnuG7kfoT5bh-bn_SfOQ5Pc,22 +jedi/third_party/typeshed/third_party/3/six/moves/builtins.pyi,sha256=VVjpGGLJ2CYwC3lYePGY6TLTEhwcdru3YV-nV2ZAzL8,23 +jedi/third_party/typeshed/third_party/3/six/moves/cPickle.pyi,sha256=pezOsQZrW9XS1R09Ote5u1Wtw9FHC0k8Kjp4g44_PgI,21 +jedi/third_party/typeshed/third_party/3/six/moves/collections_abc.pyi,sha256=9kznK-Qq5Rqt-V--bq6awTNs5NISIlTFiOZLVI4BvBA,30 +jedi/third_party/typeshed/third_party/3/six/moves/configparser.pyi,sha256=Wp5Y7Z134PHLahfawLJDB7WyIBpdLooaGKLQmEr7veQ,27 +jedi/third_party/typeshed/third_party/3/six/moves/email_mime_base.pyi,sha256=WcWEleCKHROrfdXpRuKABrT_Va1hx90NY_kxYeul3Sk,30 +jedi/third_party/typeshed/third_party/3/six/moves/email_mime_multipart.pyi,sha256=HRKWFU9qh95-mEE22_2NzEKL6lx7ynvhcfHjUcYWuZ8,35 +jedi/third_party/typeshed/third_party/3/six/moves/email_mime_nonmultipart.pyi,sha256=n5hD7R_rktJj3hiHYzEqr3CJCHSW4ikfObKHmUrXBw0,38 +jedi/third_party/typeshed/third_party/3/six/moves/email_mime_text.pyi,sha256=M7mb9V3f5JUut8yf8UfL3rG4XPr-Lr692DGjk1OR9d4,30 +jedi/third_party/typeshed/third_party/3/six/moves/html_entities.pyi,sha256=YkFcpA-UjTm7ps8gp1Xs6Ye9eu-fRHUlSrZPc00LZuk,28 +jedi/third_party/typeshed/third_party/3/six/moves/html_parser.pyi,sha256=EhnBFGx0nBd-ZHMy53ihoemWud0xnNYYYzQDrqWZ7SM,26 +jedi/third_party/typeshed/third_party/3/six/moves/http_client.pyi,sha256=a-UAXTgUTrJNFFiQWThbgVvOsqCJXXiFTxjOG4QgbiE,26 +jedi/third_party/typeshed/third_party/3/six/moves/http_cookiejar.pyi,sha256=_qfFwqs5DnvAOqLWCAdCzWjnwVFi2tkRjypRcow1Kgw,29 +jedi/third_party/typeshed/third_party/3/six/moves/http_cookies.pyi,sha256=dKSPvohzW_QPkOUb0gxj3rsshfRDYb9krTqjID3wN68,27 +jedi/third_party/typeshed/third_party/3/six/moves/queue.pyi,sha256=_rNUYjj1lkl5pRaQP4GWCuWEHBSetCgHhvSnWjgBuhk,20 +jedi/third_party/typeshed/third_party/3/six/moves/reprlib.pyi,sha256=gzyGHWv3b10R17IbpgllskSTyulpq6RWGb7I5KAbSh0,22 +jedi/third_party/typeshed/third_party/3/six/moves/socketserver.pyi,sha256=GWp7BzDMpq3JNfA3H3Pn0iyENzAcy5ufcvuvlkEzmFg,27 +jedi/third_party/typeshed/third_party/3/six/moves/tkinter.pyi,sha256=R-kj-ZjyE6cnPhkAhJLQIA2zyggMRHyf4azpH_WtXNo,22 +jedi/third_party/typeshed/third_party/3/six/moves/tkinter_commondialog.pyi,sha256=piW_7DIKFPiFl8awGTKEBkW-toBwMu7ySfSgxT39Qsc,35 +jedi/third_party/typeshed/third_party/3/six/moves/tkinter_constants.pyi,sha256=sB-tEEYJXZlnQEvgUxsHYFp3yyp3F7NtblS3_hRFVFM,32 +jedi/third_party/typeshed/third_party/3/six/moves/tkinter_dialog.pyi,sha256=Lk_TOa4m8kLSRZRs2-zLtgFnpbtkGcs2eu1YgCjNzmM,29 +jedi/third_party/typeshed/third_party/3/six/moves/tkinter_filedialog.pyi,sha256=znHuWqubMwXiONWP1bhNRmAXUVcHdXn9B8AqoJu4EgY,33 +jedi/third_party/typeshed/third_party/3/six/moves/tkinter_tkfiledialog.pyi,sha256=znHuWqubMwXiONWP1bhNRmAXUVcHdXn9B8AqoJu4EgY,33 +jedi/third_party/typeshed/third_party/3/six/moves/tkinter_ttk.pyi,sha256=4JCeiL-sndFy8xykanaUTbW3-ESBr4w8Dd1gOMAvrag,26 +jedi/third_party/typeshed/third_party/3/six/moves/urllib/__init__.pyi,sha256=F_1V8NcR4jGkws85IUurYLi4JnGh7_HttdVHvj8cQZM,217 +jedi/third_party/typeshed/third_party/3/six/moves/urllib/error.pyi,sha256=tOHyCWru4FB-CVZpdZ5tzT5jUW7R6e1NHrm_AOEx5Zo,116 +jedi/third_party/typeshed/third_party/3/six/moves/urllib/parse.pyi,sha256=LYJLXIl0_B1NV-7WuGMWjM9E12zpMbsB0SMqi4ubpR4,981 +jedi/third_party/typeshed/third_party/3/six/moves/urllib/request.pyi,sha256=Y9e1cIOCqpbA9u9wdHkuk7XkhKW4C_BR-th2joh4rKM,1639 +jedi/third_party/typeshed/third_party/3/six/moves/urllib/response.pyi,sha256=MLuhuwcVdryiGU6pB2rkOWjdFnFcm7NsXJxqFt9-YlI,389 +jedi/third_party/typeshed/third_party/3/six/moves/urllib/robotparser.pyi,sha256=WK-Nrt7QFCWmAxfbrK0Mecw9NZur54H8AoYbslX6vSg,66 +jedi/third_party/typeshed/third_party/3/six/moves/urllib_error.pyi,sha256=ZLiDEtiqtoYYbNDYF4LjnxKRd_uFft6Yi5QQyNEkZm8,27 +jedi/third_party/typeshed/third_party/3/six/moves/urllib_parse.pyi,sha256=PQR8avzMMvUSLV96WLv3J4leuJpKEUBoo7vDzP6M848,27 +jedi/third_party/typeshed/third_party/3/six/moves/urllib_request.pyi,sha256=8WFe7ycArSuM6wJfgcXWLDRKNsymd0UlxWlflszb2yk,30 +jedi/third_party/typeshed/third_party/3/six/moves/urllib_response.pyi,sha256=dokFMleMVEVFVxBgSkrcn4f4yM7RhR3zkk0iDQGOC_U,31 +jedi/third_party/typeshed/third_party/3/six/moves/urllib_robotparser.pyi,sha256=BiNO0kuoX9quQRDQsnPLr04VZLHOj57CmrJJN5OuBn4,33 +jedi/third_party/typeshed/third_party/3/typed_ast/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jedi/third_party/typeshed/third_party/3/typed_ast/ast27.pyi,sha256=8l8h1iTl2STMguvaHgJ6LObWVTN-vi7jkq7F7grliuY,6949 +jedi/third_party/typeshed/third_party/3/typed_ast/ast3.pyi,sha256=rBHvYHpYGPkPYcSCvtsS5PvfUScfOXa5NLzA0sLnjDg,7946 +jedi/third_party/typeshed/third_party/3/typed_ast/conversions.pyi,sha256=fNBvV3U3qM1ehj0f9uO9xPT9XL43AiT7v51N2S39Qf0,71 +jedi/third_party/typeshed/third_party/3/waitress/__init__.pyi,sha256=AyhjsF-JhGwHdZo_BHPZnAqQg5p6I4yrQ-GAzbIgiCE,308 +jedi/third_party/typeshed/third_party/3/waitress/adjustments.pyi,sha256=Ci0sGEaI8sgN3oQ4m63cI7_fk6b9YcQCle1SEIpkMLo,2162 +jedi/third_party/typeshed/third_party/3/waitress/buffers.pyi,sha256=BAUmNMbRRvJmXspos4XTqxpGuPcG-m8_52-ZzP1RsBU,2179 +jedi/third_party/typeshed/third_party/3/waitress/channel.pyi,sha256=yEVhDA-LraBWtHs-Qh1btontCRgfNj1crB76Mj1UeVM,1843 +jedi/third_party/typeshed/third_party/3/waitress/compat.pyi,sha256=DEqpWPIWRpAINum_lKdh8-FvKsx-8J-TnGcubrxzA6k,719 +jedi/third_party/typeshed/third_party/3/waitress/parser.pyi,sha256=xxkLG3DSOAG0gsULfgKnIDWRqDPgznvkSPm4XtD6c4o,1442 +jedi/third_party/typeshed/third_party/3/waitress/proxy_headers.pyi,sha256=dg-1E5AEYcDAdcehvBlLT_3ATo94vfS2f8GZwBZ46do,1100 +jedi/third_party/typeshed/third_party/3/waitress/receiver.pyi,sha256=ZOcx0IC-LJC3W0BwO2cZfeP7xzlFEm9ESca3oxsIm_4,1044 +jedi/third_party/typeshed/third_party/3/waitress/rfc7230.pyi,sha256=Q31Np-HnKfD9_-8UHRr4KvzdviLI4HjaIacvMKQGBFc,226 +jedi/third_party/typeshed/third_party/3/waitress/runner.pyi,sha256=ReRs3WT9zH73aXEgBq9O7e_YHDwvTlhtsZ2G-m302Hk,469 +jedi/third_party/typeshed/third_party/3/waitress/server.pyi,sha256=BLVopN0ySFAhfTjaSg-9ctygNpoMxToxGNzog_2eF2E,3500 +jedi/third_party/typeshed/third_party/3/waitress/task.pyi,sha256=uZEpvbFNjJjGtzx9HMt2vgRG1VhgzUIT7aoEN3roOGk,2216 +jedi/third_party/typeshed/third_party/3/waitress/trigger.pyi,sha256=BJ6rU86VdsB281MqGmHwUkyvf661Za4H_qhSHi5nivU,1079 +jedi/third_party/typeshed/third_party/3/waitress/utilities.pyi,sha256=jUJUiahQzp3KRB3zgfX1Zx1ebKgK-81Nz7e32ez-s4E,1875 +jedi/third_party/typeshed/third_party/3/waitress/wasyncore.pyi,sha256=mOMVSFL7majEQSv_8fEMKMGqtAXrGEqipcZ_lkQHCOY,4059 +jedi/utils.py,sha256=UDLfGghM2nrkbs1GIQR_aslkMrctH1TzOgZoMFUxtc4,4680 diff --git a/lib/python3.12/site-packages/jedi-0.19.2.dist-info/WHEEL b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ef99c6cf3283b50a273ac4c6d009a0aa85597070 --- /dev/null +++ b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.34.2) +Root-Is-Purelib: true +Tag: py2-none-any +Tag: py3-none-any + diff --git a/lib/python3.12/site-packages/jedi-0.19.2.dist-info/top_level.txt b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..86c1cb19e2763e3c2e7bab63fc0971722ead7fdf --- /dev/null +++ b/lib/python3.12/site-packages/jedi-0.19.2.dist-info/top_level.txt @@ -0,0 +1 @@ +jedi diff --git a/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/INSTALLER b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/METADATA b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..ba828d2ee3ae2f249d3522ba707b8eb46cf582d7 --- /dev/null +++ b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/METADATA @@ -0,0 +1,170 @@ +Metadata-Version: 2.4 +Name: jsonschema +Version: 4.26.0 +Summary: An implementation of JSON Schema validation for Python +Project-URL: Homepage, https://github.com/python-jsonschema/jsonschema +Project-URL: Documentation, https://python-jsonschema.readthedocs.io/ +Project-URL: Issues, https://github.com/python-jsonschema/jsonschema/issues/ +Project-URL: Funding, https://github.com/sponsors/Julian +Project-URL: Tidelift, https://tidelift.com/subscription/pkg/pypi-jsonschema?utm_source=pypi-jsonschema&utm_medium=referral&utm_campaign=pypi-link +Project-URL: Changelog, https://github.com/python-jsonschema/jsonschema/blob/main/CHANGELOG.rst +Project-URL: Source, https://github.com/python-jsonschema/jsonschema +Author-email: Julian Berman +License-Expression: MIT +License-File: COPYING +Keywords: data validation,json,json schema,jsonschema,validation +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: File Formats :: JSON +Classifier: Topic :: File Formats :: JSON :: JSON Schema +Requires-Python: >=3.10 +Requires-Dist: attrs>=22.2.0 +Requires-Dist: jsonschema-specifications>=2023.03.6 +Requires-Dist: referencing>=0.28.4 +Requires-Dist: rpds-py>=0.25.0 +Provides-Extra: format +Requires-Dist: fqdn; extra == 'format' +Requires-Dist: idna; extra == 'format' +Requires-Dist: isoduration; extra == 'format' +Requires-Dist: jsonpointer>1.13; extra == 'format' +Requires-Dist: rfc3339-validator; extra == 'format' +Requires-Dist: rfc3987; extra == 'format' +Requires-Dist: uri-template; extra == 'format' +Requires-Dist: webcolors>=1.11; extra == 'format' +Provides-Extra: format-nongpl +Requires-Dist: fqdn; extra == 'format-nongpl' +Requires-Dist: idna; extra == 'format-nongpl' +Requires-Dist: isoduration; extra == 'format-nongpl' +Requires-Dist: jsonpointer>1.13; extra == 'format-nongpl' +Requires-Dist: rfc3339-validator; extra == 'format-nongpl' +Requires-Dist: rfc3986-validator>0.1.0; extra == 'format-nongpl' +Requires-Dist: rfc3987-syntax>=1.1.0; extra == 'format-nongpl' +Requires-Dist: uri-template; extra == 'format-nongpl' +Requires-Dist: webcolors>=24.6.0; extra == 'format-nongpl' +Description-Content-Type: text/x-rst + +========== +jsonschema +========== + +|PyPI| |Pythons| |CI| |ReadTheDocs| |Precommit| |Zenodo| + +.. |PyPI| image:: https://img.shields.io/pypi/v/jsonschema.svg + :alt: PyPI version + :target: https://pypi.org/project/jsonschema/ + +.. |Pythons| image:: https://img.shields.io/pypi/pyversions/jsonschema.svg + :alt: Supported Python versions + :target: https://pypi.org/project/jsonschema/ + +.. |CI| image:: https://github.com/python-jsonschema/jsonschema/workflows/CI/badge.svg + :alt: Build status + :target: https://github.com/python-jsonschema/jsonschema/actions?query=workflow%3ACI + +.. |ReadTheDocs| image:: https://readthedocs.org/projects/python-jsonschema/badge/?version=stable&style=flat + :alt: ReadTheDocs status + :target: https://python-jsonschema.readthedocs.io/en/stable/ + +.. |Precommit| image:: https://results.pre-commit.ci/badge/github/python-jsonschema/jsonschema/main.svg + :alt: pre-commit.ci status + :target: https://results.pre-commit.ci/latest/github/python-jsonschema/jsonschema/main + +.. |Zenodo| image:: https://zenodo.org/badge/3072629.svg + :alt: Zenodo DOI + :target: https://zenodo.org/badge/latestdoi/3072629 + + +``jsonschema`` is an implementation of the `JSON Schema `_ specification for Python. + +.. code:: python + + >>> from jsonschema import validate + + >>> # A sample schema, like what we'd get from json.load() + >>> schema = { + ... "type" : "object", + ... "properties" : { + ... "price" : {"type" : "number"}, + ... "name" : {"type" : "string"}, + ... }, + ... } + + >>> # If no exception is raised by validate(), the instance is valid. + >>> validate(instance={"name" : "Eggs", "price" : 34.99}, schema=schema) + + >>> validate( + ... instance={"name" : "Eggs", "price" : "Invalid"}, schema=schema, + ... ) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValidationError: 'Invalid' is not of type 'number' + +It can also be used from the command line by installing `check-jsonschema `_. + +Features +-------- + +* Full support for `Draft 2020-12 `_, `Draft 2019-09 `_, `Draft 7 `_, `Draft 6 `_, `Draft 4 `_ and `Draft 3 `_ + +* `Lazy validation `_ that can iteratively report *all* validation errors. + +* `Programmatic querying `_ of which properties or items failed validation. + + +Installation +------------ + +``jsonschema`` is available on `PyPI `_. You can install using `pip `_: + +.. code:: bash + + $ pip install jsonschema + + +Extras +====== + +Two extras are available when installing the package, both currently related to ``format`` validation: + + * ``format`` + * ``format-nongpl`` + +They can be used when installing in order to include additional dependencies, e.g.: + +.. code:: bash + + $ pip install jsonschema'[format]' + +Be aware that the mere presence of these dependencies – or even the specification of ``format`` checks in a schema – do *not* activate format checks (as per the specification). +Please read the `format validation documentation `_ for further details. + +About +----- + +I'm Julian Berman. + +``jsonschema`` is on `GitHub `_. + +Get in touch, via GitHub or otherwise, if you've got something to contribute, it'd be most welcome! + +If you feel overwhelmingly grateful, you can also `sponsor me `_. + +And for companies who appreciate ``jsonschema`` and its continued support and growth, ``jsonschema`` is also now supportable via `TideLift `_. + + +Release Information +------------------- + +v4.26.0 +======= + +* Decrease import time by delaying importing of ``urllib.request`` (#1416). diff --git a/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/RECORD b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..c7a62bfac1ec5d0c466c77efbac096cea10145e4 --- /dev/null +++ b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/RECORD @@ -0,0 +1,82 @@ +../../../bin/jsonschema,sha256=7K2LtDrpgEFLixd7rrIPWmEqmmEaa0TVb9O5zYAdId4,249 +jsonschema-4.26.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +jsonschema-4.26.0.dist-info/METADATA,sha256=BbGJhmypipbaOTS_a4NofO0_-FMlJzt34WtwX22LJtI,7592 +jsonschema-4.26.0.dist-info/RECORD,, +jsonschema-4.26.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87 +jsonschema-4.26.0.dist-info/entry_points.txt,sha256=vO7rX4Fs_xIVJy2pnAtKgTSxfpnozAVQ0DjCmpMxnWE,51 +jsonschema-4.26.0.dist-info/licenses/COPYING,sha256=T5KgFaE8TRoEC-8BiqE0MLTxvHO0Gxa7hGw0Z2bedDk,1057 +jsonschema/__init__.py,sha256=p-Rw4TS_0OPHZIJyImDWsdWgmd6CPWHMXLq7BuQxTGc,3941 +jsonschema/__main__.py,sha256=iLsZf2upUB3ilBKTlMnyK-HHt2Cnnfkwwxi_c6gLvSA,115 +jsonschema/__pycache__/__init__.cpython-312.pyc,, +jsonschema/__pycache__/__main__.cpython-312.pyc,, +jsonschema/__pycache__/_format.cpython-312.pyc,, +jsonschema/__pycache__/_keywords.cpython-312.pyc,, +jsonschema/__pycache__/_legacy_keywords.cpython-312.pyc,, +jsonschema/__pycache__/_types.cpython-312.pyc,, +jsonschema/__pycache__/_typing.cpython-312.pyc,, +jsonschema/__pycache__/_utils.cpython-312.pyc,, +jsonschema/__pycache__/cli.cpython-312.pyc,, +jsonschema/__pycache__/exceptions.cpython-312.pyc,, +jsonschema/__pycache__/protocols.cpython-312.pyc,, +jsonschema/__pycache__/validators.cpython-312.pyc,, +jsonschema/_format.py,sha256=lrs1U7iSB2ZX9Z2EalsROYeQm1d9kefw2CLo08G8P4s,15568 +jsonschema/_keywords.py,sha256=r8_DrqAfn6QLwQnmXEggveiSU-UaIL2p2nuPINelfFc,14949 +jsonschema/_legacy_keywords.py,sha256=2tWuwRPWbYS7EAl8wBIC_rabGuv1J4dfYLqNEPpShhA,15191 +jsonschema/_types.py,sha256=f_nkTDbrja3E3fsINTC73y_gMXTU210I_H7qK0NN30c,5456 +jsonschema/_typing.py,sha256=pXLnGy3jNtIUTq037kUfq0we17085eHtJwDJ2t--mQE,617 +jsonschema/_utils.py,sha256=Xv6_wKKslBJlwyj9-j2c8JDFw-4z4aWFnVe2pX8h7U4,10659 +jsonschema/benchmarks/__init__.py,sha256=A0sQrxDBVHSyQ-8ru3L11hMXf3q9gVuB9x_YgHb4R9M,70 +jsonschema/benchmarks/__pycache__/__init__.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/const_vs_enum.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/contains.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/import_benchmark.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/issue232.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/json_schema_test_suite.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/nested_schemas.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/subcomponents.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/unused_registry.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/useless_applicator_schemas.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/useless_keywords.cpython-312.pyc,, +jsonschema/benchmarks/__pycache__/validator_creation.cpython-312.pyc,, +jsonschema/benchmarks/const_vs_enum.py,sha256=DVFi3WDqBalZFOibnjpX1uTSr3Rxa2cPgFcowd7Ukrs,830 +jsonschema/benchmarks/contains.py,sha256=gexQoUrCOwECofbt19BeosQZ7WFL6PDdkX49DWwBlOg,786 +jsonschema/benchmarks/import_benchmark.py,sha256=2DRIOVS6p-bqLn_822B_X5FkIRvsJ2V1CYnV2druzDs,779 +jsonschema/benchmarks/issue232.py,sha256=3LLYLIlBGQnVuyyo2iAv-xky5P6PRFHANx4-zIIQOoE,521 +jsonschema/benchmarks/issue232/issue.json,sha256=eaPOZjMRu5u8RpKrsA9uk7ucPZS5tkKG4D_hkOTQ3Hk,117105 +jsonschema/benchmarks/json_schema_test_suite.py,sha256=PvfabpUYcF4_7csYDTcTauED8rnFEGYbdY5RqTXD08s,320 +jsonschema/benchmarks/nested_schemas.py,sha256=mo07dx-CIgmSOI62CNs4g5xu1FzHklLBpkQoDxWYcKs,1892 +jsonschema/benchmarks/subcomponents.py,sha256=fEyiMzsWeK2pd7DEGCuuY-vzGunwhHczRBWEnBRLKIo,1113 +jsonschema/benchmarks/unused_registry.py,sha256=hwRwONc9cefPtYzkoX_TYRO3GyUojriv0-YQaK3vnj0,940 +jsonschema/benchmarks/useless_applicator_schemas.py,sha256=EVm5-EtOEFoLP_Vt2j4SrCwlx05NhPqNuZQ6LIMP1Dc,3342 +jsonschema/benchmarks/useless_keywords.py,sha256=bj_zKr1oVctFlqyZaObCsYTgFjiiNgPzC0hr1Y868mE,867 +jsonschema/benchmarks/validator_creation.py,sha256=UkUQlLAnussnr_KdCIdad6xx2pXxQLmYtsXoiirKeWQ,285 +jsonschema/cli.py,sha256=av90OtpSxuiko3FAyEHtxpI-NSuX3WMtoQpIx09obJY,8445 +jsonschema/exceptions.py,sha256=b42hUDOfPFcprI4ZlNpjDeLKv8k9vOhgWct2jyDlxzk,15256 +jsonschema/protocols.py,sha256=lgyryVHqFijSFRAz95ch3VN2Fz-oTFyhj9Obcy5P_CI,7198 +jsonschema/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jsonschema/tests/__pycache__/__init__.cpython-312.pyc,, +jsonschema/tests/__pycache__/_suite.cpython-312.pyc,, +jsonschema/tests/__pycache__/fuzz_validate.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_cli.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_deprecations.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_exceptions.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_format.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_jsonschema_test_suite.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_types.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_utils.cpython-312.pyc,, +jsonschema/tests/__pycache__/test_validators.cpython-312.pyc,, +jsonschema/tests/_suite.py,sha256=2k0X91N7dOHhQc5mrYv40OKf1weioj6RMBqWgLT6-PI,8374 +jsonschema/tests/fuzz_validate.py,sha256=fUA7yTJIihaCwJplkUehZeyB84HcXEcqtY5oPJXIO7I,1114 +jsonschema/tests/test_cli.py,sha256=A89r5LOHy-peLPZA5YDkOaMTWqzQO_w2Tu8WFz_vphM,28544 +jsonschema/tests/test_deprecations.py,sha256=yG6mkRJHpTHbWoxpLC5y5H7fk8erGOs8f_9V4tCBEh8,15754 +jsonschema/tests/test_exceptions.py,sha256=rsEqdjknUz3dEyJb6_DiModPHXsJTEWzDrxUscg8G5o,24301 +jsonschema/tests/test_format.py,sha256=eVm5SMaWF2lOPO28bPAwNvkiQvHCQKy-MnuAgEchfEc,3188 +jsonschema/tests/test_jsonschema_test_suite.py,sha256=tAfxknM65OR9LyDPHu1pkEaombLgjRLnJ6FPiWPdxjg,8461 +jsonschema/tests/test_types.py,sha256=cF51KTDmdsx06MrIc4fXKt0X9fIsVgw5uhT8CamVa8U,6977 +jsonschema/tests/test_utils.py,sha256=sao74o1PyYMxBfqweokQN48CFSS6yhJk5FkCfMJ5PsI,4163 +jsonschema/tests/test_validators.py,sha256=uzl-agyLoZPTufouIHSxFPfPYrdIdjNwjHcD_5NyU4E,87949 +jsonschema/tests/typing/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +jsonschema/tests/typing/__pycache__/__init__.cpython-312.pyc,, +jsonschema/tests/typing/__pycache__/test_all_concrete_validators_match_protocol.cpython-312.pyc,, +jsonschema/tests/typing/test_all_concrete_validators_match_protocol.py,sha256=I5XUl5ZYUSZo3APkF10l8Mnfk09oXKqvXWttGGd3sDk,1238 +jsonschema/validators.py,sha256=l_FxuZW0sMoFUwqTZqElL7F1P5nzy3sQaLbZqrEJb6U,47159 diff --git a/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/WHEEL b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..ae8ec1bdaa94d726ceb907542d76cbd5d38cafcd --- /dev/null +++ b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.28.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/entry_points.txt b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..eecef9d8fad351de6b048532776160696a796ef9 --- /dev/null +++ b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/entry_points.txt @@ -0,0 +1,2 @@ +[console_scripts] +jsonschema = jsonschema.cli:main diff --git a/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/licenses/COPYING b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/licenses/COPYING new file mode 100644 index 0000000000000000000000000000000000000000..af9cfbdb134f42e5205ecbad597421d778826481 --- /dev/null +++ b/lib/python3.12/site-packages/jsonschema-4.26.0.dist-info/licenses/COPYING @@ -0,0 +1,19 @@ +Copyright (c) 2013 Julian Berman + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/lib/python3.12/site-packages/markdown-3.10.1.dist-info/INSTALLER b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/markdown-3.10.1.dist-info/METADATA b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..5be7085a3bb6dbd68b929cab1a89a19aaaa6b870 --- /dev/null +++ b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/METADATA @@ -0,0 +1,115 @@ +Metadata-Version: 2.4 +Name: Markdown +Version: 3.10.1 +Summary: Python implementation of John Gruber's Markdown. +Author: Manfred Stienstra, Yuri Takhteyev +Author-email: Waylan limberg +Maintainer: Isaac Muse +Maintainer-email: Waylan Limberg +License-Expression: BSD-3-Clause +Project-URL: Homepage, https://Python-Markdown.github.io/ +Project-URL: Documentation, https://Python-Markdown.github.io/ +Project-URL: Repository, https://github.com/Python-Markdown/markdown +Project-URL: Issue Tracker, https://github.com/Python-Markdown/markdown/issues +Project-URL: Changelog, https://python-markdown.github.io/changelog/ +Keywords: markdown,markdown-parser,python-markdown,markdown-to-html +Classifier: Development Status :: 5 - Production/Stable +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Communications :: Email :: Filters +Classifier: Topic :: Internet :: WWW/HTTP :: Dynamic Content :: CGI Tools/Libraries +Classifier: Topic :: Internet :: WWW/HTTP :: Site Management +Classifier: Topic :: Software Development :: Documentation +Classifier: Topic :: Software Development :: Libraries :: Python Modules +Classifier: Topic :: Text Processing :: Filters +Classifier: Topic :: Text Processing :: Markup :: HTML +Classifier: Topic :: Text Processing :: Markup :: Markdown +Requires-Python: >=3.10 +Description-Content-Type: text/markdown +License-File: LICENSE.md +Provides-Extra: testing +Requires-Dist: coverage; extra == "testing" +Requires-Dist: pyyaml; extra == "testing" +Provides-Extra: docs +Requires-Dist: mkdocs>=1.6; extra == "docs" +Requires-Dist: mkdocs-nature>=0.6; extra == "docs" +Requires-Dist: mdx_gh_links>=0.2; extra == "docs" +Requires-Dist: mkdocstrings[python]>=0.28.3; extra == "docs" +Requires-Dist: mkdocs-gen-files; extra == "docs" +Requires-Dist: mkdocs-section-index; extra == "docs" +Requires-Dist: mkdocs-literate-nav; extra == "docs" +Dynamic: license-file + +[Python-Markdown][] +=================== + +[![Build Status][build-button]][build] +[![Coverage Status][codecov-button]][codecov] +[![Latest Version][mdversion-button]][md-pypi] +[![Python Versions][pyversion-button]][md-pypi] +[![BSD License][bsdlicense-button]][bsdlicense] +[![Code of Conduct][codeofconduct-button]][Code of Conduct] + +[build-button]: https://github.com/Python-Markdown/markdown/actions/workflows/tox.yml/badge.svg +[build]: https://github.com/Python-Markdown/markdown/actions/workflows/tox.yml +[codecov-button]: https://codecov.io/gh/Python-Markdown/markdown/branch/master/graph/badge.svg +[codecov]: https://codecov.io/gh/Python-Markdown/markdown +[mdversion-button]: https://img.shields.io/pypi/v/Markdown.svg +[md-pypi]: https://pypi.org/project/Markdown/ +[pyversion-button]: https://img.shields.io/pypi/pyversions/Markdown.svg +[bsdlicense-button]: https://img.shields.io/badge/license-BSD-yellow.svg +[bsdlicense]: https://opensource.org/licenses/BSD-3-Clause +[codeofconduct-button]: https://img.shields.io/badge/code%20of%20conduct-contributor%20covenant-green.svg?style=flat-square +[Code of Conduct]: https://github.com/Python-Markdown/markdown/blob/master/CODE_OF_CONDUCT.md + +This is a Python implementation of John Gruber's [Markdown][]. +It is almost completely compliant with the reference implementation, +though there are a few known issues. See [Features][] for information +on what exactly is supported and what is not. Additional features are +supported by the [Available Extensions][]. + +[Python-Markdown]: https://Python-Markdown.github.io/ +[Markdown]: https://daringfireball.net/projects/markdown/ +[Features]: https://Python-Markdown.github.io#Features +[Available Extensions]: https://Python-Markdown.github.io/extensions + +Documentation +------------- + +```bash +pip install markdown +``` +```python +import markdown +html = markdown.markdown(your_text_string) +``` + +For more advanced [installation] and [usage] documentation, see the `docs/` directory +of the distribution or the project website at . + +[installation]: https://python-markdown.github.io/install/ +[usage]: https://python-markdown.github.io/reference/ + +See the change log at . + +Support +------- + +You may report bugs, ask for help, and discuss various other issues on the [bug tracker][]. + +[bug tracker]: https://github.com/Python-Markdown/markdown/issues + +Code of Conduct +--------------- + +Everyone interacting in the Python-Markdown project's code bases, issue trackers, +and mailing lists is expected to follow the [Code of Conduct]. diff --git a/lib/python3.12/site-packages/markdown-3.10.1.dist-info/RECORD b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..3c43bea3e0e9e1341ef32699c0e8e1e36a7ed041 --- /dev/null +++ b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/RECORD @@ -0,0 +1,74 @@ +../../../bin/markdown_py,sha256=k4EeprHytJbDOjycs6xeM8-O0IZFflcbydqht16zkaY,250 +markdown-3.10.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +markdown-3.10.1.dist-info/METADATA,sha256=GkhR5MmG-PLfsz6Of7yZT-51jGgjailB_wufavu8uIw,5067 +markdown-3.10.1.dist-info/RECORD,, +markdown-3.10.1.dist-info/WHEEL,sha256=qELbo2s1Yzl39ZmrAibXA2jjPLUYfnVhUNTlyF1rq0Y,92 +markdown-3.10.1.dist-info/entry_points.txt,sha256=lMEyiiA_ZZyfPCBlDviBl-SiU0cfoeuEKpwxw361sKQ,1102 +markdown-3.10.1.dist-info/licenses/LICENSE.md,sha256=e6TrbRCzKy0R3OE4ITQDUc27swuozMZ4Qdsv_Ybnmso,1650 +markdown-3.10.1.dist-info/top_level.txt,sha256=IAxs8x618RXoH1uCqeLLxXsDefJvE_mIibr_M4sOlyk,9 +markdown/__init__.py,sha256=dfzwwdpG9L8QLEPBpLFPIHx_BN056aZXp9xZifTxYIU,1777 +markdown/__main__.py,sha256=Yk5K0Ld5N0tF9J_Q0ZC7zoWlSABYnpKqDALgTydY6ZU,5838 +markdown/__meta__.py,sha256=_87ZykgqWwiRoV0qlOgJcbKnUPCOU3ptqBU6JyyTqHQ,1713 +markdown/__pycache__/__init__.cpython-312.pyc,, +markdown/__pycache__/__main__.cpython-312.pyc,, +markdown/__pycache__/__meta__.cpython-312.pyc,, +markdown/__pycache__/blockparser.cpython-312.pyc,, +markdown/__pycache__/blockprocessors.cpython-312.pyc,, +markdown/__pycache__/core.cpython-312.pyc,, +markdown/__pycache__/htmlparser.cpython-312.pyc,, +markdown/__pycache__/inlinepatterns.cpython-312.pyc,, +markdown/__pycache__/postprocessors.cpython-312.pyc,, +markdown/__pycache__/preprocessors.cpython-312.pyc,, +markdown/__pycache__/serializers.cpython-312.pyc,, +markdown/__pycache__/test_tools.cpython-312.pyc,, +markdown/__pycache__/treeprocessors.cpython-312.pyc,, +markdown/__pycache__/util.cpython-312.pyc,, +markdown/blockparser.py,sha256=j4CQImVpiq7g9pz8wCxvzT61X_T2iSAjXupHJk8P3eA,5728 +markdown/blockprocessors.py,sha256=koY5rq8DixzBCHcquvZJp6x2JYyBGjrwxMWNZhd6D2U,27013 +markdown/core.py,sha256=sxxpCLuqS09TrIezeOMyRyj34KE-_L-Zl8mDBhbfQgE,21328 +markdown/extensions/__init__.py,sha256=9z1khsdKCVrmrJ_2GfxtPAdjD3FyMe5vhC7wmM4O9m0,4822 +markdown/extensions/__pycache__/__init__.cpython-312.pyc,, +markdown/extensions/__pycache__/abbr.cpython-312.pyc,, +markdown/extensions/__pycache__/admonition.cpython-312.pyc,, +markdown/extensions/__pycache__/attr_list.cpython-312.pyc,, +markdown/extensions/__pycache__/codehilite.cpython-312.pyc,, +markdown/extensions/__pycache__/def_list.cpython-312.pyc,, +markdown/extensions/__pycache__/extra.cpython-312.pyc,, +markdown/extensions/__pycache__/fenced_code.cpython-312.pyc,, +markdown/extensions/__pycache__/footnotes.cpython-312.pyc,, +markdown/extensions/__pycache__/legacy_attrs.cpython-312.pyc,, +markdown/extensions/__pycache__/legacy_em.cpython-312.pyc,, +markdown/extensions/__pycache__/md_in_html.cpython-312.pyc,, +markdown/extensions/__pycache__/meta.cpython-312.pyc,, +markdown/extensions/__pycache__/nl2br.cpython-312.pyc,, +markdown/extensions/__pycache__/sane_lists.cpython-312.pyc,, +markdown/extensions/__pycache__/smarty.cpython-312.pyc,, +markdown/extensions/__pycache__/tables.cpython-312.pyc,, +markdown/extensions/__pycache__/toc.cpython-312.pyc,, +markdown/extensions/__pycache__/wikilinks.cpython-312.pyc,, +markdown/extensions/abbr.py,sha256=EmeD1EZVRjNxANtDiTCyi48GtGuKxHKIw2A8hop_Gr4,7144 +markdown/extensions/admonition.py,sha256=Vj1VkMdoK1gX6_79HSkT0mCcnXiOHeEASNwL0dYfVVs,6566 +markdown/extensions/attr_list.py,sha256=BX3ltqF5dvL2ouScX8_0OKBwkC2oQugueEd89hNRPqE,7841 +markdown/extensions/codehilite.py,sha256=FVVXs5ABX-B7_ucasse1RqWMMsAxXhmXXtMgQd3e2C8,13506 +markdown/extensions/def_list.py,sha256=mQGPIIRJT8eLpnxNWvuLCe4luA67Es1QZ5ZChN7vUyA,4033 +markdown/extensions/extra.py,sha256=1vleT284kued4HQBtF83IjSumJVo0q3ng6MjTkVNfNQ,2163 +markdown/extensions/fenced_code.py,sha256=pdeC-e0hIuk60_-A9u9eiv7EqRSVLp3b7qZfCySu7bs,8303 +markdown/extensions/footnotes.py,sha256=hsH4RSBygGEi2xMWCPTSxUT76GVrH9fkvI_03klx3_c,19373 +markdown/extensions/legacy_attrs.py,sha256=oWcyNrfP0F6zsBoBOaD5NiwrJyy4kCpgQLl12HA7JGU,2788 +markdown/extensions/legacy_em.py,sha256=-Z_w4PEGSS-Xg-2-BtGAnXwwy5g5GDgv2tngASnPgxg,1693 +markdown/extensions/md_in_html.py,sha256=LM3FeNErfJTE6F125bCYSzsQZgkFTCmilN4OZq2jN2M,19785 +markdown/extensions/meta.py,sha256=Da6jDcY4esvdfLUt_MkxyPchHReTE9kcq1_Vo5IpxP8,2604 +markdown/extensions/nl2br.py,sha256=qO5FUnrkAHPtCfHfFbY5Cbm1hVhW7Ci3TRZ__8Ezxic,1102 +markdown/extensions/sane_lists.py,sha256=ttYhHruV3Jbr1z9stnWyHKrChkRSV_EwQqhwRlwIAes,2154 +markdown/extensions/smarty.py,sha256=M3D015BcvXdKi_DOhhZU1EQK8edZYVs0sO02C9JRokE,11238 +markdown/extensions/tables.py,sha256=nu_kf5H2YIEFz0218XWDsg1mQM8NGmkd10C-Ay3TldQ,8743 +markdown/extensions/toc.py,sha256=vg5s5E8yDxo1KtC75KCFGX1SJ8NWb56IHUkEwLnu4Jc,18332 +markdown/extensions/wikilinks.py,sha256=JBmBHkUzT8764tUyAHaBrkAq30UQC9RCKEXYdikDJjs,3288 +markdown/htmlparser.py,sha256=CcTivWLBXeiCRJt0JDTzNrGqjqD9mSx-a19jVXa8BoU,17275 +markdown/inlinepatterns.py,sha256=lT3QXfaa3hoTaIfYIPdJWVcA1Y8j05YG96GGgT7qlKY,38551 +markdown/postprocessors.py,sha256=vTDjF9Cx4SXlYtUSmHgNd6TIIneO1NXZjT_DObcleA0,4493 +markdown/preprocessors.py,sha256=pq5NnHKkOSVQeIo-ajC-Yt44kvyMV97D04FBOQXctJM,3224 +markdown/serializers.py,sha256=YtAFYQoOdp_TAmYGow6nBo0eB6I-Sl4PTLdLDfQJHwQ,7174 +markdown/test_tools.py,sha256=MtN4cf3ZPDtb83wXLTol-3q3aIGRIkJ2zWr6fd-RgVE,8662 +markdown/treeprocessors.py,sha256=QtWj5MqH3dN5hf1SHgriZkV1dZSk8__unCduYuadFFk,17652 +markdown/util.py,sha256=J0ZWsvuC_71sx_uKSYkHBIA4VdijxQKY5bk63SbdKG0,13943 diff --git a/lib/python3.12/site-packages/markdown-3.10.1.dist-info/WHEEL b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..fbbd86cb272bc6e329803bf67f442f2a124caa95 --- /dev/null +++ b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.10.1) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python3.12/site-packages/markdown-3.10.1.dist-info/entry_points.txt b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/entry_points.txt new file mode 100644 index 0000000000000000000000000000000000000000..be3bd8ff245a5124f9d988c2a63053d307b6d296 --- /dev/null +++ b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/entry_points.txt @@ -0,0 +1,22 @@ +[console_scripts] +markdown_py = markdown.__main__:run + +[markdown.extensions] +abbr = markdown.extensions.abbr:AbbrExtension +admonition = markdown.extensions.admonition:AdmonitionExtension +attr_list = markdown.extensions.attr_list:AttrListExtension +codehilite = markdown.extensions.codehilite:CodeHiliteExtension +def_list = markdown.extensions.def_list:DefListExtension +extra = markdown.extensions.extra:ExtraExtension +fenced_code = markdown.extensions.fenced_code:FencedCodeExtension +footnotes = markdown.extensions.footnotes:FootnoteExtension +legacy_attrs = markdown.extensions.legacy_attrs:LegacyAttrExtension +legacy_em = markdown.extensions.legacy_em:LegacyEmExtension +md_in_html = markdown.extensions.md_in_html:MarkdownInHtmlExtension +meta = markdown.extensions.meta:MetaExtension +nl2br = markdown.extensions.nl2br:Nl2BrExtension +sane_lists = markdown.extensions.sane_lists:SaneListExtension +smarty = markdown.extensions.smarty:SmartyExtension +tables = markdown.extensions.tables:TableExtension +toc = markdown.extensions.toc:TocExtension +wikilinks = markdown.extensions.wikilinks:WikiLinkExtension diff --git a/lib/python3.12/site-packages/markdown-3.10.1.dist-info/licenses/LICENSE.md b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..6249d60ce6f8a9a2a48a0314150cf0997a4a9962 --- /dev/null +++ b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/licenses/LICENSE.md @@ -0,0 +1,30 @@ +BSD 3-Clause License + +Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.12/site-packages/markdown-3.10.1.dist-info/top_level.txt b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..0918c9768895c23af5ecf35282391d7b12ed301a --- /dev/null +++ b/lib/python3.12/site-packages/markdown-3.10.1.dist-info/top_level.txt @@ -0,0 +1 @@ +markdown diff --git a/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/INSTALLER b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/LICENSE b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..eb208dada03bbd9975633d6f42eb036c09e33125 --- /dev/null +++ b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2018, Omry Yadan +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/METADATA b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..6d3ab67e16a7dfc446c5e7f878569d4fa7e79d6e --- /dev/null +++ b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/METADATA @@ -0,0 +1,64 @@ +Metadata-Version: 2.1 +Name: omegaconf +Version: 2.3.0 +Summary: A flexible configuration library +Home-page: https://github.com/omry/omegaconf +Author: Omry Yadan +Author-email: omry@yadan.net +Keywords: yaml configuration config +Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 +Classifier: Programming Language :: Python :: 3.8 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Requires-Python: >=3.6 +Description-Content-Type: text/markdown +License-File: LICENSE +Requires-Dist: antlr4-python3-runtime (==4.9.*) +Requires-Dist: PyYAML (>=5.1.0) +Requires-Dist: dataclasses ; python_version == "3.6" + +# OmegaConf +| | Description | +| --- | --- | +| Project | [![PyPI version](https://badge.fury.io/py/omegaconf.svg)](https://badge.fury.io/py/omegaconf)[![Downloads](https://pepy.tech/badge/omegaconf/month)](https://pepy.tech/project/omegaconf?versions=1.4.*&versions=2.0.*&versions=2.1.*)![PyPI - Python Version](https://img.shields.io/pypi/pyversions/omegaconf.svg) | +| Code quality| [![CircleCI](https://img.shields.io/circleci/build/github/omry/omegaconf?logo=s&token=5de2f8dc2a0dd78438520575431aa533150806e3)](https://circleci.com/gh/omry/omegaconf)[![Coverage Status](https://coveralls.io/repos/github/omry/omegaconf/badge.svg)](https://coveralls.io/github/omry/omegaconf)[![Total alerts](https://img.shields.io/lgtm/alerts/g/omry/omegaconf.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/omry/omegaconf/alerts/)[![Language grade: Python](https://img.shields.io/lgtm/grade/python/g/omry/omegaconf.svg?logo=lgtm&logoWidth=18)](https://lgtm.com/projects/g/omry/omegaconf/context:python)| +| Docs and support |[![Documentation Status](https://readthedocs.org/projects/omegaconf/badge/?version=2.0_branch)](https://omegaconf.readthedocs.io/en/2.1_branch/)[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/omry/omegaconf/master?filepath=docs%2Fnotebook%2FTutorial.ipynb)| + + +OmegaConf is a hierarchical configuration system, with support for merging configurations from multiple sources (YAML config files, dataclasses/objects and CLI arguments) +providing a consistent API regardless of how the configuration was created. + +## Releases + +### Stable (2.2) +OmegaConf 2.2 is the current stable version. +* [What's new](https://github.com/omry/omegaconf/releases/tag/v2.2.1) +* [Documentation](https://omegaconf.readthedocs.io/en/2.2_branch/) +* [Source code](https://github.com/omry/omegaconf/tree/2.2_branch) + +Install with `pip install --upgrade omegaconf` + +### Previous release (2.1) +OmegaConf 2.1 is the current stable version. +* [What's new](https://github.com/omry/omegaconf/releases/tag/v2.1.1) +* [Documentation](https://omegaconf.readthedocs.io/en/2.1_branch/) +* [Slides](https://docs.google.com/presentation/d/e/2PACX-1vT_UIV7hCnquIbLUm4NnkUpXvPEh33IKiUEvPRF850WKA8opOlZOszjKdZ3tPmf8u7hGNP6HpqS-NT5/pub?start=false&loop=false&delayms=3000) +* [Source code](https://github.com/omry/omegaconf/tree/2.1_branch) + +Install with `pip install omegaconf==2.1` + +### Previous release (2.0) + +* [What's new](https://github.com/omry/omegaconf/releases/tag/v2.0.0) +* [Documentation](https://omegaconf.readthedocs.io/en/2.0_branch/) +* [Slides](https://docs.google.com/presentation/d/e/2PACX-1vT_UIV7hCnquIbLUm4NnkUpXvPEh33IKiUEvPRF850WKA8opOlZOszjKdZ3tPmf8u7hGNP6HpqS-NT5/pub?start=false&loop=false&delayms=3000) +* [Source code](https://github.com/omry/omegaconf/tree/2.0_branch) + +Install with `pip install omegaconf==2.0.6` + +## Live tutorial +Run the live tutorial: [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/omry/omegaconf/master?filepath=docs%2Fnotebook%2FTutorial.ipynb) diff --git a/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/RECORD b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..0698d75a3b716fa8212485946bdac8b9ddb3a6ef --- /dev/null +++ b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/RECORD @@ -0,0 +1,58 @@ +omegaconf-2.3.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +omegaconf-2.3.0.dist-info/LICENSE,sha256=sxN17V1cvn7C0MTBTfivp6jlUBu-JfcLbLxGFoX0uRo,1518 +omegaconf-2.3.0.dist-info/METADATA,sha256=qy3L8OPnDMddyh2JT4lT596LZR3C3CjwjzUL-leHuts,3865 +omegaconf-2.3.0.dist-info/RECORD,, +omegaconf-2.3.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +omegaconf-2.3.0.dist-info/WHEEL,sha256=2wepM1nk4DS4eFpYrW1TTqPcoGNfHhhO_i5m4cOimbo,92 +omegaconf-2.3.0.dist-info/top_level.txt,sha256=gKo5sjlMnML5r9-l1eLYBTS1xZ-xAr67mlBSgL9yvSA,25 +omegaconf/__init__.py,sha256=lynwiDHEFBIHMqrBN3HRh0KJevzIap6MyULFNkMmlBc,1170 +omegaconf/__pycache__/__init__.cpython-312.pyc,, +omegaconf/__pycache__/_impl.cpython-312.pyc,, +omegaconf/__pycache__/_utils.cpython-312.pyc,, +omegaconf/__pycache__/base.cpython-312.pyc,, +omegaconf/__pycache__/basecontainer.cpython-312.pyc,, +omegaconf/__pycache__/dictconfig.cpython-312.pyc,, +omegaconf/__pycache__/errors.cpython-312.pyc,, +omegaconf/__pycache__/grammar_parser.cpython-312.pyc,, +omegaconf/__pycache__/grammar_visitor.cpython-312.pyc,, +omegaconf/__pycache__/listconfig.cpython-312.pyc,, +omegaconf/__pycache__/nodes.cpython-312.pyc,, +omegaconf/__pycache__/omegaconf.cpython-312.pyc,, +omegaconf/__pycache__/version.cpython-312.pyc,, +omegaconf/_impl.py,sha256=CJfcRt8K20BiMMW5u3q8wuQ9hAeAmuYlFxctzPgTIgQ,2877 +omegaconf/_utils.py,sha256=bw8Jpra0B8YkHFrwODrBcKPvgQTyU0pEpskLl7CDBRg,32564 +omegaconf/base.py,sha256=rHWuLw8Iiew_SNJIUkLZy4EfyRb60hyS9X9Hl4OuBvs,32462 +omegaconf/basecontainer.py,sha256=FWL2PZWRb7wP-AGpeN651l_uVnNhZvzzLjaUdDlJiB8,33318 +omegaconf/dictconfig.py,sha256=IPtUPHoak025Q-gvDVqMdS497Xb6tjcsawrHXv4iQ2Y,27855 +omegaconf/errors.py,sha256=ectbX8Kd-5FCJMTBIh6jshqvCnCiXMFGfFQJYGIereQ,3595 +omegaconf/grammar/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +omegaconf/grammar/__pycache__/__init__.cpython-312.pyc,, +omegaconf/grammar/gen/OmegaConfGrammarLexer.py,sha256=eoog1aofOltw-u61QFeCWZ3A9oa6vK2tjTaX4wmJY14,21689 +omegaconf/grammar/gen/OmegaConfGrammarParser.py,sha256=X0uJc0OieVB_f9yKRZw4OfwRBPOGoiyDMjhz8OKPvZ4,63429 +omegaconf/grammar/gen/OmegaConfGrammarParserListener.py,sha256=ncibh-Nhg68pdJPdMaqeFTP876wvo0lpJqPXhaOmjZU,5843 +omegaconf/grammar/gen/OmegaConfGrammarParserVisitor.py,sha256=Amvi244RtYFebNSJCbOwsqdvQW-QgdDKTku0rXnkD5Y,3617 +omegaconf/grammar/gen/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +omegaconf/grammar/gen/__pycache__/OmegaConfGrammarLexer.cpython-312.pyc,, +omegaconf/grammar/gen/__pycache__/OmegaConfGrammarParser.cpython-312.pyc,, +omegaconf/grammar/gen/__pycache__/OmegaConfGrammarParserListener.cpython-312.pyc,, +omegaconf/grammar/gen/__pycache__/OmegaConfGrammarParserVisitor.cpython-312.pyc,, +omegaconf/grammar/gen/__pycache__/__init__.cpython-312.pyc,, +omegaconf/grammar_parser.py,sha256=7gvIGsqtsgl5e0XOWhI3Ur58FoIk3iBaX_M8JdyzpYM,5511 +omegaconf/grammar_visitor.py,sha256=L6LOomG24GcPA1-aYzNOmKLup5kYirHTuyp857V7T7A,16055 +omegaconf/listconfig.py,sha256=LyNRrn84BqNiz-TpwrDILat8HhlVwwM6zFmYCTUAdIo,24671 +omegaconf/nodes.py,sha256=1XRIC8sfEoaV4PT_Vm7Z75JMsIc30kKTuJHGyZ2aM0w,17479 +omegaconf/omegaconf.py,sha256=hFeVL0k6PNRqGE7ZUpI2p6sSGPkc9IEvsqr3MNdgU_s,39048 +omegaconf/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +omegaconf/resolvers/__init__.py,sha256=zE-8WJwipGC2cXPq12CGAz49m82F1D1cKMwuV3EoT7k,60 +omegaconf/resolvers/__pycache__/__init__.cpython-312.pyc,, +omegaconf/resolvers/oc/__init__.py,sha256=gLniKaouCmkm1ZTQOzS1F5jFlCMw7eL-ae6PhaMxNjg,3548 +omegaconf/resolvers/oc/__pycache__/__init__.cpython-312.pyc,, +omegaconf/resolvers/oc/__pycache__/dict.cpython-312.pyc,, +omegaconf/resolvers/oc/dict.py,sha256=OL7iZ_aBIt7kYXxhMiFxSyskiX-xXBvGI_DnBbdxqVs,2324 +omegaconf/version.py,sha256=9OuwPWmsrNRBQYmeipe5XnywKY4jw-ZQBrjbo5C5Md4,442 +pydevd_plugins/__init__.py,sha256=byyA3PmxQsWCnqpQzFy44heJq59wOqHr5bSt6knm5vY,180 +pydevd_plugins/__pycache__/__init__.cpython-312.pyc,, +pydevd_plugins/extensions/__init__.py,sha256=byyA3PmxQsWCnqpQzFy44heJq59wOqHr5bSt6knm5vY,180 +pydevd_plugins/extensions/__pycache__/__init__.cpython-312.pyc,, +pydevd_plugins/extensions/__pycache__/pydevd_plugin_omegaconf.cpython-312.pyc,, +pydevd_plugins/extensions/pydevd_plugin_omegaconf.py,sha256=Ac1SCniO-22CsRsMAXhFyyoVSN2R-MUWsOujm-601Ak,4251 diff --git a/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/REQUESTED b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/REQUESTED new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/WHEEL b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..57e3d840d59a650ac5bccbad5baeec47d155f0ad --- /dev/null +++ b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.38.4) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/top_level.txt b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..fcd2b36b800878631de7b82d0218569415c6a323 --- /dev/null +++ b/lib/python3.12/site-packages/omegaconf-2.3.0.dist-info/top_level.txt @@ -0,0 +1,2 @@ +omegaconf +pydevd_plugins diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/INSTALLER b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/METADATA b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..c147a6463bc3a423623f2c4cc616b82e4cd32a1b --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/METADATA @@ -0,0 +1,175 @@ +Metadata-Version: 2.4 +Name: pillow +Version: 12.1.0 +Summary: Python Imaging Library (fork) +Author-email: "Jeffrey A. Clark" +License-Expression: MIT-CMU +Project-URL: Changelog, https://github.com/python-pillow/Pillow/releases +Project-URL: Documentation, https://pillow.readthedocs.io +Project-URL: Funding, https://tidelift.com/subscription/pkg/pypi-pillow?utm_source=pypi-pillow&utm_medium=pypi +Project-URL: Homepage, https://python-pillow.github.io +Project-URL: Mastodon, https://fosstodon.org/@pillow +Project-URL: Release notes, https://pillow.readthedocs.io/en/stable/releasenotes/index.html +Project-URL: Source, https://github.com/python-pillow/Pillow +Keywords: Imaging +Classifier: Development Status :: 6 - Mature +Classifier: Programming Language :: Python :: 3 :: Only +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Programming Language :: Python :: 3.14 +Classifier: Programming Language :: Python :: Implementation :: CPython +Classifier: Programming Language :: Python :: Implementation :: PyPy +Classifier: Topic :: Multimedia :: Graphics +Classifier: Topic :: Multimedia :: Graphics :: Capture :: Digital Camera +Classifier: Topic :: Multimedia :: Graphics :: Capture :: Screen Capture +Classifier: Topic :: Multimedia :: Graphics :: Graphics Conversion +Classifier: Topic :: Multimedia :: Graphics :: Viewers +Classifier: Typing :: Typed +Requires-Python: >=3.10 +Description-Content-Type: text/markdown +License-File: LICENSE +Provides-Extra: docs +Requires-Dist: furo; extra == "docs" +Requires-Dist: olefile; extra == "docs" +Requires-Dist: sphinx>=8.2; extra == "docs" +Requires-Dist: sphinx-autobuild; extra == "docs" +Requires-Dist: sphinx-copybutton; extra == "docs" +Requires-Dist: sphinx-inline-tabs; extra == "docs" +Requires-Dist: sphinxext-opengraph; extra == "docs" +Provides-Extra: fpx +Requires-Dist: olefile; extra == "fpx" +Provides-Extra: mic +Requires-Dist: olefile; extra == "mic" +Provides-Extra: test-arrow +Requires-Dist: arro3-compute; extra == "test-arrow" +Requires-Dist: arro3-core; extra == "test-arrow" +Requires-Dist: nanoarrow; extra == "test-arrow" +Requires-Dist: pyarrow; extra == "test-arrow" +Provides-Extra: tests +Requires-Dist: check-manifest; extra == "tests" +Requires-Dist: coverage>=7.4.2; extra == "tests" +Requires-Dist: defusedxml; extra == "tests" +Requires-Dist: markdown2; extra == "tests" +Requires-Dist: olefile; extra == "tests" +Requires-Dist: packaging; extra == "tests" +Requires-Dist: pyroma>=5; extra == "tests" +Requires-Dist: pytest; extra == "tests" +Requires-Dist: pytest-cov; extra == "tests" +Requires-Dist: pytest-timeout; extra == "tests" +Requires-Dist: pytest-xdist; extra == "tests" +Requires-Dist: trove-classifiers>=2024.10.12; extra == "tests" +Provides-Extra: xmp +Requires-Dist: defusedxml; extra == "xmp" +Dynamic: license-file + +

+ Pillow logo +

+ +# Pillow + +## Python Imaging Library (Fork) + +Pillow is the friendly PIL fork by [Jeffrey A. Clark and +contributors](https://github.com/python-pillow/Pillow/graphs/contributors). +PIL is the Python Imaging Library by Fredrik Lundh and contributors. +As of 2019, Pillow development is +[supported by Tidelift](https://tidelift.com/subscription/pkg/pypi-pillow?utm_source=pypi-pillow&utm_medium=readme&utm_campaign=enterprise). + + + + + + + + + + + + + + + + + + +
docs + Documentation Status +
tests + GitHub Actions build status (Lint) + GitHub Actions build status (Test Linux and macOS) + GitHub Actions build status (Test Windows) + GitHub Actions build status (Test MinGW) + GitHub Actions build status (Test Docker) + GitHub Actions build status (Wheels) + Code coverage + Fuzzing Status +
package + Zenodo + Tidelift + Newest PyPI version + Number of PyPI downloads + OpenSSF Best Practices +
social + Join the chat at https://gitter.im/python-pillow/Pillow + Follow on https://fosstodon.org/@pillow +
+ +## Overview + +The Python Imaging Library adds image processing capabilities to your Python interpreter. + +This library provides extensive file format support, an efficient internal representation, and fairly powerful image processing capabilities. + +The core image library is designed for fast access to data stored in a few basic pixel formats. It should provide a solid foundation for a general image processing tool. + +## More information + +- [Documentation](https://pillow.readthedocs.io/) + - [Installation](https://pillow.readthedocs.io/en/latest/installation/basic-installation.html) + - [Handbook](https://pillow.readthedocs.io/en/latest/handbook/index.html) +- [Contribute](https://github.com/python-pillow/Pillow/blob/main/.github/CONTRIBUTING.md) + - [Issues](https://github.com/python-pillow/Pillow/issues) + - [Pull requests](https://github.com/python-pillow/Pillow/pulls) +- [Release notes](https://pillow.readthedocs.io/en/stable/releasenotes/index.html) +- [Changelog](https://github.com/python-pillow/Pillow/releases) + - [Pre-fork](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst#pre-fork) + +## Report a vulnerability + +To report a security vulnerability, please follow the procedure described in the [Tidelift security policy](https://tidelift.com/docs/security). diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/RECORD b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..9446e38c9cb9a901ac0cf23e501ad859cad77821 --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/RECORD @@ -0,0 +1,237 @@ +PIL/AvifImagePlugin.py,sha256=sHaYxONtVEDZ24y8Ad7qRlixD4iIPC2ipH1W79-uPbE,9030 +PIL/BdfFontFile.py,sha256=PhlZfIRmEfmorbhZZeSM5eebGo1Ei7fL-lR9XlfTZZA,3285 +PIL/BlpImagePlugin.py,sha256=tQxeJbpiyvp3-zSLjeWAQr8YdLS_HoKWmuLDQwRQl4w,16568 +PIL/BmpImagePlugin.py,sha256=64PB3YE8kEL8QrAuUMlWoEaByYJUTYV6OQA254Uj4Lc,19925 +PIL/BufrStubImagePlugin.py,sha256=gMbELNwEghE_IM2Ov1NT4jy-96CTpEvyuj_mXmckL-c,1765 +PIL/ContainerIO.py,sha256=wkBqL2GDAb5fh3wrtfTGUfqioJipCl-lg2GxbjQrTZw,4604 +PIL/CurImagePlugin.py,sha256=-WEsgwQbA9rQzXB0HG0LK1V_qbuwHosPZ0T2IjfN8r0,1791 +PIL/DcxImagePlugin.py,sha256=OlCv6yDI0LLZNbuUI7s69FI1DuArPyGxo3YCzQ1n8Hg,2180 +PIL/DdsImagePlugin.py,sha256=JRhDGFhKXvolAiD_X4_nYQ3DARogu2hmE2AhZuGc02Q,18931 +PIL/EpsImagePlugin.py,sha256=jnT9Y01VEOdnU0Q3ZncyeTM81WYPpPABXT2BOgKW-mY,16626 +PIL/ExifTags.py,sha256=zW6kVikCosiyoCo7J7R62evD3hoxjKPchnVh8po7CZc,9931 +PIL/FitsImagePlugin.py,sha256=-oDJnAH113CK5qPvwz9lL81fkV1gla_tNfqLcq8zKgo,4644 +PIL/FliImagePlugin.py,sha256=4zxH8IXBX9DGi6dJRM6Y5NMdbA1d99x696mcGZHxHzI,4929 +PIL/FontFile.py,sha256=St7MxO5Q-oakCLWn3ZrgrtaT3wSsmAarxm8AU-G8Moc,3577 +PIL/FpxImagePlugin.py,sha256=NnHQj6Ze1LUXqIOi7WHCKBEBftAkJZJJ1quHEpZaE14,7363 +PIL/FtexImagePlugin.py,sha256=ZS0WqBHtItNoi0W52A4REOGrDIfh40dCGDLruaXLRFE,3570 +PIL/GbrImagePlugin.py,sha256=DYyvhekFB1sOQLEWPwYbf0CEfGdgdax1ci-3DuSeIgQ,3053 +PIL/GdImageFile.py,sha256=LP4Uxv3Y2ivGZIyOVuGJarDDVS7zK6F1Q6SNl4wyGuQ,2788 +PIL/GifImagePlugin.py,sha256=Bu5YTg7YwLgJIxcE0RiDGHIGTApmGZxXvWQn0dPs674,42304 +PIL/GimpGradientFile.py,sha256=gqqUkDbKVFCtBxt5VAhPS0HtLZDYFI6KWEaUhhTNNE8,3982 +PIL/GimpPaletteFile.py,sha256=hIHQ9LJ5ri0hy1e_vZYeD-n67UWdhEDlKc4vDxgaUdg,1860 +PIL/GribStubImagePlugin.py,sha256=C9YNx1hxBgYvRFzcs1ZG-srEG_3A3DIWBgk1j-nbl8I,1794 +PIL/Hdf5StubImagePlugin.py,sha256=4JirZFt94qjH73NsWJi83OP-cEUMKDTsIhWKrdNGvuc,1776 +PIL/IcnsImagePlugin.py,sha256=l3JdNkYQVAMeiGpK_tem2vtDD11KtAJ64wQGsA0E-JQ,12440 +PIL/IcoImagePlugin.py,sha256=UGzIDlQ02ZB-VJEsoyI7YWmTx26ZiB2EHJZ1qvB28JQ,13103 +PIL/ImImagePlugin.py,sha256=8sBOAy7xlqP722sXWhnJn4VoBhTBzbZQTtfW_XsWefU,11602 +PIL/Image.py,sha256=Pl7NzD6IAHSZAeYcgQi9PEkGPTukRPoLYrDBivAiAH8,148899 +PIL/ImageChops.py,sha256=GEjlymcoDtA5OOeIxQVIX96BD-s6AXhb7TmSLYn2tUg,7946 +PIL/ImageCms.py,sha256=IuCm3gXKpb5Eu1kn-TB8cD9XJLZEa8fpkEjzVqAIKNk,40676 +PIL/ImageColor.py,sha256=IGA9C2umeED_EzS2Cvj6KsU0VutC9RstWIYPe8uDsVk,9441 +PIL/ImageDraw.py,sha256=FMn0AK_gxxJBYB8afOGF2FUP2KJPFvUf3UZp_KrJz7A,36287 +PIL/ImageDraw2.py,sha256=pdVMW7bVw3KwhXvRZh28Md4y-2xFfuo5fHcDnaYqVK4,7227 +PIL/ImageEnhance.py,sha256=4Elhz_lyyxLmx0GkSHrwOAmNJ2TkqVQPHejzGihZUMI,3627 +PIL/ImageFile.py,sha256=W5_QULZWeY7Kqz04cLujVu0kSBnKZu7liLCCqoXO-GM,29972 +PIL/ImageFilter.py,sha256=MO1MBrbXDiX2IAGESdGm_0087bwmSZ_14ecAj28ojCY,18729 +PIL/ImageFont.py,sha256=tsIwpnCjWl57_sMcAFTt84Ikpf14EPrW8_IJ9XQ7z7E,63253 +PIL/ImageGrab.py,sha256=oaeBn7UHjtxKPXJ9OSXPUt3cG1I0j-4yesbfvh0p_TY,7750 +PIL/ImageMath.py,sha256=RQl6cRXGuszba4KwtbIudin_8U65shpWrajr9gTn1rw,10369 +PIL/ImageMode.py,sha256=aaZVHAiCEanOA2K1jN3DlW3NPKa8Dm5nIXTXErzyFms,2395 +PIL/ImageMorph.py,sha256=otf5f9A-Y4Aosfl1E4YWEVatgDBInGp6xWBJfAJ0NRs,10356 +PIL/ImageOps.py,sha256=bIcQFK_MtovfNSYTcOesp4So9OgsGrwt3cGsB7xlGRM,25567 +PIL/ImagePalette.py,sha256=uPP_qWWSTWitlv4WdXucZF4n-2ZNsHjg4H36g-JpuxU,9092 +PIL/ImagePath.py,sha256=5yUG5XCUil1KKTTA_8PgGhcmg-mnue-GK0FwTBlhjw4,371 +PIL/ImageQt.py,sha256=PTt5TPyngWL-Vuvx_bwnH17EOBe3tE7l4huVmvGQP5Y,6684 +PIL/ImageSequence.py,sha256=Mphgkr79scmYBgmi9ZguhDfVwHvpLSX5uZVHDZlrn0I,2253 +PIL/ImageShow.py,sha256=Ju0_Db2B4_n3yKJV9sDsF7_HAgciEdXlq6I1Eiw1YTo,10106 +PIL/ImageStat.py,sha256=FVTiYWGCciPW1QD61b7DYZlcDqR0dS6hsLjq-gcKcG4,5495 +PIL/ImageText.py,sha256=E3EEmIlP1FG-zgy9YddETN1HKSIXaRJSrPJIC67aAxY,12120 +PIL/ImageTk.py,sha256=b5SntckGXs0ECsI2MmdJg3CSX6AtELsWh0Ohxu41u_k,8132 +PIL/ImageTransform.py,sha256=-qek7P3lzLddcXt9cWt5w_L11JGp2yY3AJtOfmJAkDc,3916 +PIL/ImageWin.py,sha256=LT05w8_vTfRrC3n9S9pM0TNbXrzZLEJHlCJil7Xv80k,8085 +PIL/ImtImagePlugin.py,sha256=SL5IrsHcblltxtX4v_HVFhYnR6haJ0AOd2NHhZKMImY,2665 +PIL/IptcImagePlugin.py,sha256=AV5HTo7kHo20IuF1rd4OhQcbf3wt1rbRPwpI3cK0xw4,6579 +PIL/Jpeg2KImagePlugin.py,sha256=DugpqlY-L54ZSbOOr1mux_fT1APy73e9715NQx_zL9E,14002 +PIL/JpegImagePlugin.py,sha256=kWLRMNTVrfcHaIpZpcLsaLtxaefrenWdILZWwZfYEZ8,31543 +PIL/JpegPresets.py,sha256=lnqWHo4DLIHIulcdHp0NJ7CWexHt8T3w51kIKlLfkIA,12379 +PIL/McIdasImagePlugin.py,sha256=baOIkD-CIIeCgBFTf8kos928PKBuCUqYYa38u3WES_8,1877 +PIL/MicImagePlugin.py,sha256=j91UizvL-zgXZgOcq_4oWBbSX6LPq1J9kNHkdITM1Qw,2599 +PIL/MpegImagePlugin.py,sha256=g7BZd93kWpFi41SG_wKFoi0yEPsioI4kj45b2F-3Vrw,2010 +PIL/MpoImagePlugin.py,sha256=xRCA2R2lSpT76pqDiQX0Es0yyaHj5d4gdN26eV7aInU,6792 +PIL/MspImagePlugin.py,sha256=oxk_MLUDvzJ4JDuOZCHkmqOPXniG42PHOyNGwe60slY,5892 +PIL/PSDraw.py,sha256=KMBGj3vXaFpblaIcA9KjFFTpdal41AQggY-UgzqoMkQ,6918 +PIL/PaletteFile.py,sha256=suDdAL6VMljXw4oEn1vhTt4DQ4vbpIHGd3A4oxOgE6s,1216 +PIL/PalmImagePlugin.py,sha256=WJ1b8I1xTSAXYDJhIpkVFCLu2LlpbiBD5d1Hr-m2l08,8748 +PIL/PcdImagePlugin.py,sha256=-gnMUqQH0R-aljsd3nZS9eBI1j75ijWD_HZfadE3RsQ,1774 +PIL/PcfFontFile.py,sha256=DqcyydQgP2vtiPFzj57KYHLuF2v-0oMTB-VkgYYHKhE,7223 +PIL/PcxImagePlugin.py,sha256=1xAq6CdH34cOsOgTPi4Wu2SKQCdQiTLVyqaMkYQZUP4,6245 +PIL/PdfImagePlugin.py,sha256=6lZLoQMVbAE-x1ESrv6PgGSyM9Ueck7e6E6ps-YQ-vI,9321 +PIL/PdfParser.py,sha256=Hr3ImLDSIKwUF6OrQ1GjlAnGi6ZpGVLWhGfKhqQ_DRM,37996 +PIL/PixarImagePlugin.py,sha256=l_4GwBd0mATnIXYJbwmmODU2vP7wewLu6BRviHCB2EI,1758 +PIL/PngImagePlugin.py,sha256=XaVZDKJIf37Cp9tSZr74kYWFTVfV5PYZ9Qeo8pVbeqo,51721 +PIL/PpmImagePlugin.py,sha256=vb5SP0IjQPzDRDE8jSPtcJv9K3Rh1LczAlt0Pg26i90,12391 +PIL/PsdImagePlugin.py,sha256=Dxyb0AwhIk7ngpuOluaJz3HtlQtSUmylLJao9CPiCII,8720 +PIL/QoiImagePlugin.py,sha256=o3IJbJYiG6Qz_evDAixeTferNaXOA7Rf0TsoNQeFaZ4,8607 +PIL/SgiImagePlugin.py,sha256=3Ql89s8vycNWjcxJwMw28iksV9Yj2xWoKBQ6c5DHXBg,6389 +PIL/SpiderImagePlugin.py,sha256=etb3Q-rKGtBg_rrVmBfh-poM2AODqibtA3yq5zoUTSc,10306 +PIL/SunImagePlugin.py,sha256=Hdxkhk0pxpBGxYhPJfCDLwsYcO1KjxjtplNMFYibIvk,4589 +PIL/TarIO.py,sha256=BqYUChCBb9F7Sh-uZ86iz1Dtoy2D0obNwGm65z1rdc0,1442 +PIL/TgaImagePlugin.py,sha256=2vDsFTcBUBHw1V80wpVv4tgpLDbPr6yVHi6Fvaqf0HY,6980 +PIL/TiffImagePlugin.py,sha256=cIQ48x3zmm5PFSG01wqweC8DJUhJxrX1R62c8Edw1Jg,85002 +PIL/TiffTags.py,sha256=dm3qNQQkuaLKqzUTvnlGiUA8K3IizQOABvac_wjttF0,17206 +PIL/WalImageFile.py,sha256=P6Ue6JZGAwBKE-JAIUP-oHCC4MPPc2jyqOScnAEKK3Y,5761 +PIL/WebPImagePlugin.py,sha256=fCHqpMBM5CV2pr26GzLEuBm7V58Q_Z9YJ-9zxc2jA4M,9854 +PIL/WmfImagePlugin.py,sha256=B2pMdJxRildCITQdb4XNAJXEwjitr2ZZVKVl2c-G1xY,5316 +PIL/XVThumbImagePlugin.py,sha256=XghmnRm8Y_8j0O28XPbHRrmK7CrGVuR844gv1hfF5ug,2126 +PIL/XbmImagePlugin.py,sha256=Fd6GVDEo73nyFICA3Z3w4LjkwoZWvhHB6rKCm5yVrho,2669 +PIL/XpmImagePlugin.py,sha256=jtUKavJCYwIAsJaJwSx8vJsx1oTbCywfDxePENmA93w,4400 +PIL/__init__.py,sha256=Q4KOEpR7S_Xsj30fvOsvR94xEpX4KUsVeUwaVP1fU80,2031 +PIL/__main__.py,sha256=Lpj4vef8mI7jA1sRCUAoVYaeePD_Uc898xF5c7XLx1A,133 +PIL/__pycache__/AvifImagePlugin.cpython-312.pyc,, +PIL/__pycache__/BdfFontFile.cpython-312.pyc,, +PIL/__pycache__/BlpImagePlugin.cpython-312.pyc,, +PIL/__pycache__/BmpImagePlugin.cpython-312.pyc,, +PIL/__pycache__/BufrStubImagePlugin.cpython-312.pyc,, +PIL/__pycache__/ContainerIO.cpython-312.pyc,, +PIL/__pycache__/CurImagePlugin.cpython-312.pyc,, +PIL/__pycache__/DcxImagePlugin.cpython-312.pyc,, +PIL/__pycache__/DdsImagePlugin.cpython-312.pyc,, +PIL/__pycache__/EpsImagePlugin.cpython-312.pyc,, +PIL/__pycache__/ExifTags.cpython-312.pyc,, +PIL/__pycache__/FitsImagePlugin.cpython-312.pyc,, +PIL/__pycache__/FliImagePlugin.cpython-312.pyc,, +PIL/__pycache__/FontFile.cpython-312.pyc,, +PIL/__pycache__/FpxImagePlugin.cpython-312.pyc,, +PIL/__pycache__/FtexImagePlugin.cpython-312.pyc,, +PIL/__pycache__/GbrImagePlugin.cpython-312.pyc,, +PIL/__pycache__/GdImageFile.cpython-312.pyc,, +PIL/__pycache__/GifImagePlugin.cpython-312.pyc,, +PIL/__pycache__/GimpGradientFile.cpython-312.pyc,, +PIL/__pycache__/GimpPaletteFile.cpython-312.pyc,, +PIL/__pycache__/GribStubImagePlugin.cpython-312.pyc,, +PIL/__pycache__/Hdf5StubImagePlugin.cpython-312.pyc,, +PIL/__pycache__/IcnsImagePlugin.cpython-312.pyc,, +PIL/__pycache__/IcoImagePlugin.cpython-312.pyc,, +PIL/__pycache__/ImImagePlugin.cpython-312.pyc,, +PIL/__pycache__/Image.cpython-312.pyc,, +PIL/__pycache__/ImageChops.cpython-312.pyc,, +PIL/__pycache__/ImageCms.cpython-312.pyc,, +PIL/__pycache__/ImageColor.cpython-312.pyc,, +PIL/__pycache__/ImageDraw.cpython-312.pyc,, +PIL/__pycache__/ImageDraw2.cpython-312.pyc,, +PIL/__pycache__/ImageEnhance.cpython-312.pyc,, +PIL/__pycache__/ImageFile.cpython-312.pyc,, +PIL/__pycache__/ImageFilter.cpython-312.pyc,, +PIL/__pycache__/ImageFont.cpython-312.pyc,, +PIL/__pycache__/ImageGrab.cpython-312.pyc,, +PIL/__pycache__/ImageMath.cpython-312.pyc,, +PIL/__pycache__/ImageMode.cpython-312.pyc,, +PIL/__pycache__/ImageMorph.cpython-312.pyc,, +PIL/__pycache__/ImageOps.cpython-312.pyc,, +PIL/__pycache__/ImagePalette.cpython-312.pyc,, +PIL/__pycache__/ImagePath.cpython-312.pyc,, +PIL/__pycache__/ImageQt.cpython-312.pyc,, +PIL/__pycache__/ImageSequence.cpython-312.pyc,, +PIL/__pycache__/ImageShow.cpython-312.pyc,, +PIL/__pycache__/ImageStat.cpython-312.pyc,, +PIL/__pycache__/ImageText.cpython-312.pyc,, +PIL/__pycache__/ImageTk.cpython-312.pyc,, +PIL/__pycache__/ImageTransform.cpython-312.pyc,, +PIL/__pycache__/ImageWin.cpython-312.pyc,, +PIL/__pycache__/ImtImagePlugin.cpython-312.pyc,, +PIL/__pycache__/IptcImagePlugin.cpython-312.pyc,, +PIL/__pycache__/Jpeg2KImagePlugin.cpython-312.pyc,, +PIL/__pycache__/JpegImagePlugin.cpython-312.pyc,, +PIL/__pycache__/JpegPresets.cpython-312.pyc,, +PIL/__pycache__/McIdasImagePlugin.cpython-312.pyc,, +PIL/__pycache__/MicImagePlugin.cpython-312.pyc,, +PIL/__pycache__/MpegImagePlugin.cpython-312.pyc,, +PIL/__pycache__/MpoImagePlugin.cpython-312.pyc,, +PIL/__pycache__/MspImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PSDraw.cpython-312.pyc,, +PIL/__pycache__/PaletteFile.cpython-312.pyc,, +PIL/__pycache__/PalmImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PcdImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PcfFontFile.cpython-312.pyc,, +PIL/__pycache__/PcxImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PdfImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PdfParser.cpython-312.pyc,, +PIL/__pycache__/PixarImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PngImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PpmImagePlugin.cpython-312.pyc,, +PIL/__pycache__/PsdImagePlugin.cpython-312.pyc,, +PIL/__pycache__/QoiImagePlugin.cpython-312.pyc,, +PIL/__pycache__/SgiImagePlugin.cpython-312.pyc,, +PIL/__pycache__/SpiderImagePlugin.cpython-312.pyc,, +PIL/__pycache__/SunImagePlugin.cpython-312.pyc,, +PIL/__pycache__/TarIO.cpython-312.pyc,, +PIL/__pycache__/TgaImagePlugin.cpython-312.pyc,, +PIL/__pycache__/TiffImagePlugin.cpython-312.pyc,, +PIL/__pycache__/TiffTags.cpython-312.pyc,, +PIL/__pycache__/WalImageFile.cpython-312.pyc,, +PIL/__pycache__/WebPImagePlugin.cpython-312.pyc,, +PIL/__pycache__/WmfImagePlugin.cpython-312.pyc,, +PIL/__pycache__/XVThumbImagePlugin.cpython-312.pyc,, +PIL/__pycache__/XbmImagePlugin.cpython-312.pyc,, +PIL/__pycache__/XpmImagePlugin.cpython-312.pyc,, +PIL/__pycache__/__init__.cpython-312.pyc,, +PIL/__pycache__/__main__.cpython-312.pyc,, +PIL/__pycache__/_binary.cpython-312.pyc,, +PIL/__pycache__/_deprecate.cpython-312.pyc,, +PIL/__pycache__/_tkinter_finder.cpython-312.pyc,, +PIL/__pycache__/_typing.cpython-312.pyc,, +PIL/__pycache__/_util.cpython-312.pyc,, +PIL/__pycache__/_version.cpython-312.pyc,, +PIL/__pycache__/features.cpython-312.pyc,, +PIL/__pycache__/report.cpython-312.pyc,, +PIL/_avif.cpython-312-x86_64-linux-gnu.so,sha256=yRNgSy1InaliqpKaMpMB9kHUhxRQbzzTpnlR2l6oEQ0,87889 +PIL/_avif.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +PIL/_binary.py,sha256=pcM6AL04GxgmGeLfcH1V1BZHENwIrQH0uxhJ7r0HIL0,2550 +PIL/_deprecate.py,sha256=KhyLVmQYVVww0rORbU24Te3t8fOt9G2LeM6fJGWbl0o,2034 +PIL/_imaging.cpython-312-x86_64-linux-gnu.so,sha256=WT6eaMKR4ryop_lRX7pGD8o6YcFeJtMZ1ohBy7A_OMM,3357545 +PIL/_imaging.pyi,sha256=fT-TTGQS0kym1gv77gXTKQtJrAmO1NynS5LKBtG3G6M,893 +PIL/_imagingcms.cpython-312-x86_64-linux-gnu.so,sha256=7McplYI5qWt3rMmA8Cv35YGQ1yzlIreG8KdRzvBgiDY,157841 +PIL/_imagingcms.pyi,sha256=ZZ8iIoi6EHWLvgAdfm1hPD5CQmxi75LiJl5x8yGxYoU,4433 +PIL/_imagingft.cpython-312-x86_64-linux-gnu.so,sha256=2Hwh6PkylrG_moiTQHO9HxD5SeNRzrfJTrKo-kDtDds,322977 +PIL/_imagingft.pyi,sha256=cYySzvcKBCiHPBsvttMie9AdfUcEsqZR-3256YQtz2Q,1833 +PIL/_imagingmath.cpython-312-x86_64-linux-gnu.so,sha256=vn7EvKt8b6fEk0cA-j8OmBVnhCC-a2IaWIFKB-58Dbo,167536 +PIL/_imagingmath.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +PIL/_imagingmorph.cpython-312-x86_64-linux-gnu.so,sha256=CM4oTAEjGiUXu6TWMCA5kM_hJQEONl-KVvcQ6umrod0,37040 +PIL/_imagingmorph.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +PIL/_imagingtk.cpython-312-x86_64-linux-gnu.so,sha256=lYneIS1JCAqNQd0cjp-B4fXn3cId7RMzdUc4ORcEyJE,47000 +PIL/_imagingtk.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +PIL/_tkinter_finder.py,sha256=GIZ4stmFhUosmHKSrdxcjStiocDNfyJn7RBie2SWxU0,538 +PIL/_typing.py,sha256=2z33ZUp9aQnkSqXzNR3Zn7l04d2W-oAj1OiZhiyFF68,919 +PIL/_util.py,sha256=fxhWdrLARyc2PsMgN3m9_U1dY3oUKbV7mkoHcXgoeeA,684 +PIL/_version.py,sha256=1FqYx4jdgQYvEAvLtump96L4wA1kWOGXfGf-HEnbqpE,87 +PIL/_webp.cpython-312-x86_64-linux-gnu.so,sha256=cOm91PbOITaPei2DOEswpa6UxpEj1cxiL4sVwSeL-z4,108849 +PIL/_webp.pyi,sha256=3fBxcSppJr6EOEcUojvflG3Eegg7lv2Qp0dNQQILrP4,63 +PIL/features.py,sha256=FPkEhjtBaRSqpkgHNYduwxiFtycu4NjZKwEMWxtemPU,10775 +PIL/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +PIL/report.py,sha256=4JY6-IU7sH1RKuRbOvy1fUt0dAoi79FX4tYJN3p1DT0,100 +pillow-12.1.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +pillow-12.1.0.dist-info/METADATA,sha256=Tn_tqaVNfW_nHRGmPzygXdXnDb9QCmz3yoJnJ3TF_Is,8808 +pillow-12.1.0.dist-info/RECORD,, +pillow-12.1.0.dist-info/WHEEL,sha256=1crAxrAH5rUbvWUY1UR0ly3o7KnT1jo0_98V8RY5-FM,152 +pillow-12.1.0.dist-info/licenses/LICENSE,sha256=MBeL96_5-NyCr-01CGzTeKkGTnf8tDgEhfOLXaM3cFI,68061 +pillow-12.1.0.dist-info/sboms/auditwheel.cdx.json,sha256=eB9cR7ov38Ek2TBREeiO2JCFIxmpsl9qjuX-M3iy9tE,1358 +pillow-12.1.0.dist-info/top_level.txt,sha256=riZqrk-hyZqh5f1Z0Zwii3dKfxEsByhu9cU9IODF-NY,4 +pillow-12.1.0.dist-info/zip-safe,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1 +pillow.libs/libXau-154567c4.so.6.0.0,sha256=BUhNJL94y47QMWnxywZyBNgpy3ryHeiCBADSnRFeQyA,22081 +pillow.libs/libavif-01e67780.so.16.3.0,sha256=xCXlA8_rggzsCA_EsWKFzgqPBmT1ihpZCo3iMyvD1P0,5142057 +pillow.libs/libbrotlicommon-c55a5f7a.so.1.2.0,sha256=KRd3RkM3AIqxjtuqeOUvrXmS2ThJ7qT9_g5ML8pRfI8,144425 +pillow.libs/libbrotlidec-b57ddf63.so.1.2.0,sha256=ZBDWekS9g0GRgRJS3t6vIAZioIC1lqQfinrBbT1Ml4E,62337 +pillow.libs/libfreetype-ee1c40c4.so.6.20.4,sha256=NT-L0LUGjsSLzTo-_0Cd-Vkpkfa9n8nrrqMm-Ynk-Ao,1463609 +pillow.libs/libharfbuzz-0692f733.so.0.61230.0,sha256=Vk9HU3ouaHMJ_VFtuhDv4EfRq6R6XjzSAWKmz2_nyd0,932841 +pillow.libs/libjpeg-18a3f47f.so.62.4.0,sha256=j0qE2uSWrTShO98KcxnTOWnpPPQfDaXhWpiUBLXZwBw,836273 +pillow.libs/liblcms2-cc10e42f.so.2.0.17,sha256=5JMjEDVKMwxcinhbQl6qhRLaezAiQFYEPSz-KultHe0,519073 +pillow.libs/liblzma-61b1002e.so.5.8.2,sha256=8WWs0fVFL4eJdo0YS7JQyClvQogXEpZnYmtcAdbKBZQ,266369 +pillow.libs/libopenjp2-94e588ba.so.2.5.4,sha256=5ye2mwzPYSo447VzscBGO_ZWXyBgFEPFRY45hlXLIw0,585849 +pillow.libs/libpng16-4a38ea05.so.16.53.0,sha256=3RgdzzWJ0lAiCEDXTJb6VOVy_Uz2uIEJz3iBrd-l6l8,278001 +pillow.libs/libsharpyuv-95d8a097.so.0.1.2,sha256=EtR2hzr_XVKswxcXrVFfDixKeUd1TMu56F0oHwH3Els,46113 +pillow.libs/libtiff-295fd75c.so.6.2.0,sha256=ThM3Vmh2AeUvQPNnxorCgEoVVxj05AdRjFmA1G_4Upo,754729 +pillow.libs/libwebp-d8b9687f.so.7.2.0,sha256=k-gbdtoXnzmBItYhvYxdhV11ff08TsQgWSMUUHmMzPs,731209 +pillow.libs/libwebpdemux-747f2b49.so.2.0.17,sha256=jsiJz7rjNfyn9TOqFT2OPydmTYax0iNBCCBQg9st9vw,30217 +pillow.libs/libwebpmux-7f11e5ce.so.3.1.2,sha256=Pes9BQ-MFyCzlQH9n07pWNtt7O0afgVlBsMJ-6kuU_o,58617 +pillow.libs/libxcb-64009ff3.so.1.1.0,sha256=t0N-0WuuesRJgEn9FOENG9HD59FdDl6rHS6tQqg6SdE,251425 +pillow.libs/libzstd-761a17b6.so.1.5.7,sha256=jKEGQObGqZaFbxpkGZX8jzstJVcHhJPh4SShnGgZjB8,1800497 diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/WHEEL b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..895655c2dd100ef008b7c91dd73244160a062179 --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/WHEEL @@ -0,0 +1,6 @@ +Wheel-Version: 1.0 +Generator: setuptools (80.9.0) +Root-Is-Purelib: false +Tag: cp312-cp312-manylinux_2_27_x86_64 +Tag: cp312-cp312-manylinux_2_28_x86_64 + diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/licenses/LICENSE b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/licenses/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..a5ad83b9b79dbb611fd7f68dce93043625c305be --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/licenses/LICENSE @@ -0,0 +1,1523 @@ +The Python Imaging Library (PIL) is + + Copyright © 1997-2011 by Secret Labs AB + Copyright © 1995-2011 by Fredrik Lundh and contributors + +Pillow is the friendly PIL fork. It is + + Copyright © 2010 by Jeffrey A. Clark and contributors + +Like PIL, Pillow is licensed under the open source MIT-CMU License: + +By obtaining, using, and/or copying this software and/or its associated +documentation, you agree that you have read, understood, and will comply +with the following terms and conditions: + +Permission to use, copy, modify and distribute this software and its +documentation for any purpose and without fee is hereby granted, +provided that the above copyright notice appears in all copies, and that +both that copyright notice and this permission notice appear in supporting +documentation, and that the name of Secret Labs AB or the author not be +used in advertising or publicity pertaining to distribution of the software +without specific, written prior permission. + +SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS +SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. +IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR BE LIABLE FOR ANY SPECIAL, +INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM +LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE +OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR +PERFORMANCE OF THIS SOFTWARE. + + +---- + +AOM + +Copyright (c) 2016, Alliance for Open Media. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +---- + +BROTLI + +Copyright (c) 2009, 2010, 2013-2016 by the Brotli Authors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +---- + +BZIP2 + + +-------------------------------------------------------------------------- + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org +bzip2/libbzip2 version 1.0.8 of 13 July 2019 + +-------------------------------------------------------------------------- + + +---- + +DAV1D + +Copyright © 2018-2019, VideoLAN and dav1d authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +---- + +FREETYPE2 + +The FreeType 2 font engine is copyrighted work and cannot be used +legally without a software license. In order to make this project +usable to a vast majority of developers, we distribute it under two +mutually exclusive open-source licenses. + +This means that *you* must choose *one* of the two licenses described +below, then obey all its terms and conditions when using FreeType 2 in +any of your projects or products. + + - The FreeType License, found in the file `docs/FTL.TXT`, which is + similar to the original BSD license *with* an advertising clause + that forces you to explicitly cite the FreeType project in your + product's documentation. All details are in the license file. + This license is suited to products which don't use the GNU General + Public License. + + Note that this license is compatible to the GNU General Public + License version 3, but not version 2. + + - The GNU General Public License version 2, found in + `docs/GPLv2.TXT` (any later version can be used also), for + programs which already use the GPL. Note that the FTL is + incompatible with GPLv2 due to its advertisement clause. + +The contributed BDF and PCF drivers come with a license similar to +that of the X Window System. It is compatible to the above two +licenses (see files `src/bdf/README` and `src/pcf/README`). The same +holds for the source code files `src/base/fthash.c` and +`include/freetype/internal/fthash.h`; they were part of the BDF driver +in earlier FreeType versions. + +The gzip module uses the zlib license (see `src/gzip/zlib.h`) which +too is compatible to the above two licenses. + +The files `src/autofit/ft-hb.c` and `src/autofit/ft-hb.h` contain code +taken almost verbatim from the HarfBuzz file `hb-ft.cc`, which uses +the 'Old MIT' license, compatible to the above two licenses. + +The MD5 checksum support (only used for debugging in development +builds) is in the public domain. + +-------------------------------------------------------------------------- + + The FreeType Project LICENSE + ---------------------------- + + 2006-Jan-27 + + Copyright 1996-2002, 2006 by + David Turner, Robert Wilhelm, and Werner Lemberg + + + +Introduction +============ + + The FreeType Project is distributed in several archive packages; + some of them may contain, in addition to the FreeType font engine, + various tools and contributions which rely on, or relate to, the + FreeType Project. + + This license applies to all files found in such packages, and + which do not fall under their own explicit license. The license + affects thus the FreeType font engine, the test programs, + documentation and makefiles, at the very least. + + This license was inspired by the BSD, Artistic, and IJG + (Independent JPEG Group) licenses, which all encourage inclusion + and use of free software in commercial and freeware products + alike. As a consequence, its main points are that: + + o We don't promise that this software works. However, we will be + interested in any kind of bug reports. (`as is' distribution) + + o You can use this software for whatever you want, in parts or + full form, without having to pay us. (`royalty-free' usage) + + o You may not pretend that you wrote this software. If you use + it, or only parts of it, in a program, you must acknowledge + somewhere in your documentation that you have used the + FreeType code. (`credits') + + We specifically permit and encourage the inclusion of this + software, with or without modifications, in commercial products. + We disclaim all warranties covering The FreeType Project and + assume no liability related to The FreeType Project. + + + Finally, many people asked us for a preferred form for a + credit/disclaimer to use in compliance with this license. We thus + encourage you to use the following text: + + """ + Portions of this software are copyright © The FreeType + Project (www.freetype.org). All rights reserved. + """ + + Please replace with the value from the FreeType version you + actually use. + + +Legal Terms +=========== + +0. Definitions +-------------- + + Throughout this license, the terms `package', `FreeType Project', + and `FreeType archive' refer to the set of files originally + distributed by the authors (David Turner, Robert Wilhelm, and + Werner Lemberg) as the `FreeType Project', be they named as alpha, + beta or final release. + + `You' refers to the licensee, or person using the project, where + `using' is a generic term including compiling the project's source + code as well as linking it to form a `program' or `executable'. + This program is referred to as `a program using the FreeType + engine'. + + This license applies to all files distributed in the original + FreeType Project, including all source code, binaries and + documentation, unless otherwise stated in the file in its + original, unmodified form as distributed in the original archive. + If you are unsure whether or not a particular file is covered by + this license, you must contact us to verify this. + + The FreeType Project is copyright (C) 1996-2000 by David Turner, + Robert Wilhelm, and Werner Lemberg. All rights reserved except as + specified below. + +1. No Warranty +-------------- + + THE FREETYPE PROJECT IS PROVIDED `AS IS' WITHOUT WARRANTY OF ANY + KIND, EITHER EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, + WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE. IN NO EVENT WILL ANY OF THE AUTHORS OR COPYRIGHT HOLDERS + BE LIABLE FOR ANY DAMAGES CAUSED BY THE USE OR THE INABILITY TO + USE, OF THE FREETYPE PROJECT. + +2. Redistribution +----------------- + + This license grants a worldwide, royalty-free, perpetual and + irrevocable right and license to use, execute, perform, compile, + display, copy, create derivative works of, distribute and + sublicense the FreeType Project (in both source and object code + forms) and derivative works thereof for any purpose; and to + authorize others to exercise some or all of the rights granted + herein, subject to the following conditions: + + o Redistribution of source code must retain this license file + (`FTL.TXT') unaltered; any additions, deletions or changes to + the original files must be clearly indicated in accompanying + documentation. The copyright notices of the unaltered, + original files must be preserved in all copies of source + files. + + o Redistribution in binary form must provide a disclaimer that + states that the software is based in part of the work of the + FreeType Team, in the distribution documentation. We also + encourage you to put an URL to the FreeType web page in your + documentation, though this isn't mandatory. + + These conditions apply to any software derived from or based on + the FreeType Project, not just the unmodified files. If you use + our work, you must acknowledge us. However, no fee need be paid + to us. + +3. Advertising +-------------- + + Neither the FreeType authors and contributors nor you shall use + the name of the other for commercial, advertising, or promotional + purposes without specific prior written permission. + + We suggest, but do not require, that you use one or more of the + following phrases to refer to this software in your documentation + or advertising materials: `FreeType Project', `FreeType Engine', + `FreeType library', or `FreeType Distribution'. + + As you have not signed this license, you are not required to + accept it. However, as the FreeType Project is copyrighted + material, only this license, or another one contracted with the + authors, grants you the right to use, distribute, and modify it. + Therefore, by using, distributing, or modifying the FreeType + Project, you indicate that you understand and accept all the terms + of this license. + +4. Contacts +----------- + + There are two mailing lists related to FreeType: + + o freetype@nongnu.org + + Discusses general use and applications of FreeType, as well as + future and wanted additions to the library and distribution. + If you are looking for support, start in this list if you + haven't found anything to help you in the documentation. + + o freetype-devel@nongnu.org + + Discusses bugs, as well as engine internals, design issues, + specific licenses, porting, etc. + + Our home page can be found at + + https://www.freetype.org + + +--- end of FTL.TXT --- + +The following license details are part of `src/bdf/README`: + +``` +License +******* + +Copyright (C) 2001-2002 by Francesco Zappa Nardelli + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +*** Portions of the driver (that is, bdflib.c and bdf.h): + +Copyright 2000 Computing Research Labs, New Mexico State University +Copyright 2001-2002, 2011 Francesco Zappa Nardelli + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL +THE COMPUTING RESEARCH LAB OR NEW MEXICO STATE UNIVERSITY BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT +OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR +THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Credits +******* + +This driver is based on excellent Mark Leisher's bdf library. If you +find something good in this driver you should probably thank him, not +me. +``` + +The following license details are part of `src/pcf/README`: + +``` +License +******* + +Copyright (C) 2000 by Francesco Zappa Nardelli + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +Credits +******* + +Keith Packard wrote the pcf driver found in XFree86. His work is at +the same time the specification and the sample implementation of the +PCF format. Undoubtedly, this driver is inspired from his work. +``` + + +---- + +HARFBUZZ + +HarfBuzz is licensed under the so-called "Old MIT" license. Details follow. +For parts of HarfBuzz that are licensed under different licenses see individual +files names COPYING in subdirectories where applicable. + +Copyright © 2010-2022 Google, Inc. +Copyright © 2015-2020 Ebrahim Byagowi +Copyright © 2019,2020 Facebook, Inc. +Copyright © 2012,2015 Mozilla Foundation +Copyright © 2011 Codethink Limited +Copyright © 2008,2010 Nokia Corporation and/or its subsidiary(-ies) +Copyright © 2009 Keith Stribley +Copyright © 2011 Martin Hosken and SIL International +Copyright © 2007 Chris Wilson +Copyright © 2005,2006,2020,2021,2022,2023 Behdad Esfahbod +Copyright © 2004,2007,2008,2009,2010,2013,2021,2022,2023 Red Hat, Inc. +Copyright © 1998-2005 David Turner and Werner Lemberg +Copyright © 2016 Igalia S.L. +Copyright © 2022 Matthias Clasen +Copyright © 2018,2021 Khaled Hosny +Copyright © 2018,2019,2020 Adobe, Inc +Copyright © 2013-2015 Alexei Podtelezhnikov + +For full copyright notices consult the individual files in the package. + + +Permission is hereby granted, without written agreement and without +license or royalty fees, to use, copy, modify, and distribute this +software and its documentation for any purpose, provided that the +above copyright notice and the following two paragraphs appear in +all copies of this software. + +IN NO EVENT SHALL THE COPYRIGHT HOLDER BE LIABLE TO ANY PARTY FOR +DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES +ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN +IF THE COPYRIGHT HOLDER HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +THE COPYRIGHT HOLDER SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, +BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND +FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS +ON AN "AS IS" BASIS, AND THE COPYRIGHT HOLDER HAS NO OBLIGATION TO +PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + + +---- + +LCMS2 + +Little CMS +Copyright (c) 1998-2020 Marti Maria Saguer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +---- + +LIBAVIF + +Copyright 2019 Joe Drago. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------ + +Files: src/obu.c + +Copyright © 2018-2019, VideoLAN and dav1d authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------ + +Files: third_party/iccjpeg/* + +In plain English: + +1. We don't promise that this software works. (But if you find any bugs, + please let us know!) +2. You can use this software for whatever you want. You don't have to pay us. +3. You may not pretend that you wrote this software. If you use it in a + program, you must acknowledge somewhere in your documentation that + you've used the IJG code. + +In legalese: + +The authors make NO WARRANTY or representation, either express or implied, +with respect to this software, its quality, accuracy, merchantability, or +fitness for a particular purpose. This software is provided "AS IS", and you, +its user, assume the entire risk as to its quality and accuracy. + +This software is copyright (C) 1991-2013, Thomas G. Lane, Guido Vollbeding. +All Rights Reserved except as specified below. + +Permission is hereby granted to use, copy, modify, and distribute this +software (or portions thereof) for any purpose, without fee, subject to these +conditions: +(1) If any part of the source code for this software is distributed, then this +README file must be included, with this copyright and no-warranty notice +unaltered; and any additions, deletions, or changes to the original files +must be clearly indicated in accompanying documentation. +(2) If only executable code is distributed, then the accompanying +documentation must state that "this software is based in part on the work of +the Independent JPEG Group". +(3) Permission for use of this software is granted only if the user accepts +full responsibility for any undesirable consequences; the authors accept +NO LIABILITY for damages of any kind. + +These conditions apply to any software derived from or based on the IJG code, +not just to the unmodified library. If you use our work, you ought to +acknowledge us. + +Permission is NOT granted for the use of any IJG author's name or company name +in advertising or publicity relating to this software or products derived from +it. This software may be referred to only as "the Independent JPEG Group's +software". + +We specifically permit and encourage the use of this software as the basis of +commercial products, provided that all warranty or liability claims are +assumed by the product vendor. + + +The Unix configuration script "configure" was produced with GNU Autoconf. +It is copyright by the Free Software Foundation but is freely distributable. +The same holds for its supporting scripts (config.guess, config.sub, +ltmain.sh). Another support script, install-sh, is copyright by X Consortium +but is also freely distributable. + +The IJG distribution formerly included code to read and write GIF files. +To avoid entanglement with the Unisys LZW patent, GIF reading support has +been removed altogether, and the GIF writer has been simplified to produce +"uncompressed GIFs". This technique does not use the LZW algorithm; the +resulting GIF files are larger than usual, but are readable by all standard +GIF decoders. + +We are required to state that + "The Graphics Interchange Format(c) is the Copyright property of + CompuServe Incorporated. GIF(sm) is a Service Mark property of + CompuServe Incorporated." + +------------------------------------------------------------------------------ + +Files: contrib/gdk-pixbuf/* + +Copyright 2020 Emmanuel Gil Peyrot. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------------------------------------------------------------------ + +Files: android_jni/gradlew* + + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------------------------------------------------------------------ + +Files: third_party/libyuv/* + +Copyright 2011 The LibYuv Project Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +---- + +LIBJPEG + +1. We don't promise that this software works. (But if you find any bugs, + please let us know!) +2. You can use this software for whatever you want. You don't have to pay us. +3. You may not pretend that you wrote this software. If you use it in a + program, you must acknowledge somewhere in your documentation that + you've used the IJG code. + +In legalese: + +The authors make NO WARRANTY or representation, either express or implied, +with respect to this software, its quality, accuracy, merchantability, or +fitness for a particular purpose. This software is provided "AS IS", and you, +its user, assume the entire risk as to its quality and accuracy. + +This software is copyright (C) 1991-2020, Thomas G. Lane, Guido Vollbeding. +All Rights Reserved except as specified below. + +Permission is hereby granted to use, copy, modify, and distribute this +software (or portions thereof) for any purpose, without fee, subject to these +conditions: +(1) If any part of the source code for this software is distributed, then this +README file must be included, with this copyright and no-warranty notice +unaltered; and any additions, deletions, or changes to the original files +must be clearly indicated in accompanying documentation. +(2) If only executable code is distributed, then the accompanying +documentation must state that "this software is based in part on the work of +the Independent JPEG Group". +(3) Permission for use of this software is granted only if the user accepts +full responsibility for any undesirable consequences; the authors accept +NO LIABILITY for damages of any kind. + +These conditions apply to any software derived from or based on the IJG code, +not just to the unmodified library. If you use our work, you ought to +acknowledge us. + +Permission is NOT granted for the use of any IJG author's name or company name +in advertising or publicity relating to this software or products derived from +it. This software may be referred to only as "the Independent JPEG Group's +software". + +We specifically permit and encourage the use of this software as the basis of +commercial products, provided that all warranty or liability claims are +assumed by the product vendor. + + +---- + +LIBLZMA + +XZ Utils Licensing +================== + + Different licenses apply to different files in this package. Here + is a rough summary of which licenses apply to which parts of this + package (but check the individual files to be sure!): + + - liblzma is in the public domain. + + - xz, xzdec, and lzmadec command line tools are in the public + domain unless GNU getopt_long had to be compiled and linked + in from the lib directory. The getopt_long code is under + GNU LGPLv2.1+. + + - The scripts to grep, diff, and view compressed files have been + adapted from gzip. These scripts and their documentation are + under GNU GPLv2+. + + - All the documentation in the doc directory and most of the + XZ Utils specific documentation files in other directories + are in the public domain. + + - Translated messages are in the public domain. + + - The build system contains public domain files, and files that + are under GNU GPLv2+ or GNU GPLv3+. None of these files end up + in the binaries being built. + + - Test files and test code in the tests directory, and debugging + utilities in the debug directory are in the public domain. + + - The extra directory may contain public domain files, and files + that are under various free software licenses. + + You can do whatever you want with the files that have been put into + the public domain. If you find public domain legally problematic, + take the previous sentence as a license grant. If you still find + the lack of copyright legally problematic, you have too many + lawyers. + + As usual, this software is provided "as is", without any warranty. + + If you copy significant amounts of public domain code from XZ Utils + into your project, acknowledging this somewhere in your software is + polite (especially if it is proprietary, non-free software), but + naturally it is not legally required. Here is an example of a good + notice to put into "about box" or into documentation: + + This software includes code from XZ Utils . + + The following license texts are included in the following files: + - COPYING.LGPLv2.1: GNU Lesser General Public License version 2.1 + - COPYING.GPLv2: GNU General Public License version 2 + - COPYING.GPLv3: GNU General Public License version 3 + + Note that the toolchain (compiler, linker etc.) may add some code + pieces that are copyrighted. Thus, it is possible that e.g. liblzma + binary wouldn't actually be in the public domain in its entirety + even though it contains no copyrighted code from the XZ Utils source + package. + + If you have questions, don't hesitate to ask the author(s) for more + information. + + +---- + +LIBPNG + +COPYRIGHT NOTICE, DISCLAIMER, and LICENSE +========================================= + +PNG Reference Library License version 2 +--------------------------------------- + + * Copyright (c) 1995-2022 The PNG Reference Library Authors. + * Copyright (c) 2018-2022 Cosmin Truta. + * Copyright (c) 2000-2002, 2004, 2006-2018 Glenn Randers-Pehrson. + * Copyright (c) 1996-1997 Andreas Dilger. + * Copyright (c) 1995-1996 Guy Eric Schalnat, Group 42, Inc. + +The software is supplied "as is", without warranty of any kind, +express or implied, including, without limitation, the warranties +of merchantability, fitness for a particular purpose, title, and +non-infringement. In no event shall the Copyright owners, or +anyone distributing the software, be liable for any damages or +other liability, whether in contract, tort or otherwise, arising +from, out of, or in connection with the software, or the use or +other dealings in the software, even if advised of the possibility +of such damage. + +Permission is hereby granted to use, copy, modify, and distribute +this software, or portions hereof, for any purpose, without fee, +subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you + must not claim that you wrote the original software. If you + use this software in a product, an acknowledgment in the product + documentation would be appreciated, but is not required. + + 2. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + + 3. This Copyright notice may not be removed or altered from any + source or altered source distribution. + + +PNG Reference Library License version 1 (for libpng 0.5 through 1.6.35) +----------------------------------------------------------------------- + +libpng versions 1.0.7, July 1, 2000, through 1.6.35, July 15, 2018 are +Copyright (c) 2000-2002, 2004, 2006-2018 Glenn Randers-Pehrson, are +derived from libpng-1.0.6, and are distributed according to the same +disclaimer and license as libpng-1.0.6 with the following individuals +added to the list of Contributing Authors: + + Simon-Pierre Cadieux + Eric S. Raymond + Mans Rullgard + Cosmin Truta + Gilles Vollant + James Yu + Mandar Sahastrabuddhe + Google Inc. + Vadim Barkov + +and with the following additions to the disclaimer: + + There is no warranty against interference with your enjoyment of + the library or against infringement. There is no warranty that our + efforts or the library will fulfill any of your particular purposes + or needs. This library is provided with all faults, and the entire + risk of satisfactory quality, performance, accuracy, and effort is + with the user. + +Some files in the "contrib" directory and some configure-generated +files that are distributed with libpng have other copyright owners, and +are released under other open source licenses. + +libpng versions 0.97, January 1998, through 1.0.6, March 20, 2000, are +Copyright (c) 1998-2000 Glenn Randers-Pehrson, are derived from +libpng-0.96, and are distributed according to the same disclaimer and +license as libpng-0.96, with the following individuals added to the +list of Contributing Authors: + + Tom Lane + Glenn Randers-Pehrson + Willem van Schaik + +libpng versions 0.89, June 1996, through 0.96, May 1997, are +Copyright (c) 1996-1997 Andreas Dilger, are derived from libpng-0.88, +and are distributed according to the same disclaimer and license as +libpng-0.88, with the following individuals added to the list of +Contributing Authors: + + John Bowler + Kevin Bracey + Sam Bushell + Magnus Holmgren + Greg Roelofs + Tom Tanner + +Some files in the "scripts" directory have other copyright owners, +but are released under this license. + +libpng versions 0.5, May 1995, through 0.88, January 1996, are +Copyright (c) 1995-1996 Guy Eric Schalnat, Group 42, Inc. + +For the purposes of this copyright and license, "Contributing Authors" +is defined as the following set of individuals: + + Andreas Dilger + Dave Martindale + Guy Eric Schalnat + Paul Schmidt + Tim Wegner + +The PNG Reference Library is supplied "AS IS". The Contributing +Authors and Group 42, Inc. disclaim all warranties, expressed or +implied, including, without limitation, the warranties of +merchantability and of fitness for any purpose. The Contributing +Authors and Group 42, Inc. assume no liability for direct, indirect, +incidental, special, exemplary, or consequential damages, which may +result from the use of the PNG Reference Library, even if advised of +the possibility of such damage. + +Permission is hereby granted to use, copy, modify, and distribute this +source code, or portions hereof, for any purpose, without fee, subject +to the following restrictions: + + 1. The origin of this source code must not be misrepresented. + + 2. Altered versions must be plainly marked as such and must not + be misrepresented as being the original source. + + 3. This Copyright notice may not be removed or altered from any + source or altered source distribution. + +The Contributing Authors and Group 42, Inc. specifically permit, +without fee, and encourage the use of this source code as a component +to supporting the PNG file format in commercial products. If you use +this source code in a product, acknowledgment is not required but would +be appreciated. + + +---- + +LIBTIFF + +Copyright (c) 1988-1997 Sam Leffler +Copyright (c) 1991-1997 Silicon Graphics, Inc. + +Permission to use, copy, modify, distribute, and sell this software and +its documentation for any purpose is hereby granted without fee, provided +that (i) the above copyright notices and this permission notice appear in +all copies of the software and related documentation, and (ii) the names of +Sam Leffler and Silicon Graphics may not be used in any advertising or +publicity relating to the software without the specific, prior written +permission of Sam Leffler and Silicon Graphics. + +THE SOFTWARE IS PROVIDED "AS-IS" AND WITHOUT WARRANTY OF ANY KIND, +EXPRESS, IMPLIED OR OTHERWISE, INCLUDING WITHOUT LIMITATION, ANY +WARRANTY OF MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. + +IN NO EVENT SHALL SAM LEFFLER OR SILICON GRAPHICS BE LIABLE FOR +ANY SPECIAL, INCIDENTAL, INDIRECT OR CONSEQUENTIAL DAMAGES OF ANY KIND, +OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +WHETHER OR NOT ADVISED OF THE POSSIBILITY OF DAMAGE, AND ON ANY THEORY OF +LIABILITY, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +OF THIS SOFTWARE. + + +---- + +LIBWEBP + +Copyright (c) 2010, Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +---- + +LIBYUV + +Copyright 2011 The LibYuv Project Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +---- + +OPENJPEG + +* + * The copyright in this software is being made available under the 2-clauses + * BSD License, included below. This software may be subject to other third + * party and contributor rights, including patent rights, and no such rights + * are granted under this license. + * + * Copyright (c) 2002-2014, Universite catholique de Louvain (UCL), Belgium + * Copyright (c) 2002-2014, Professor Benoit Macq + * Copyright (c) 2003-2014, Antonin Descampe + * Copyright (c) 2003-2009, Francois-Olivier Devaux + * Copyright (c) 2005, Herve Drolon, FreeImage Team + * Copyright (c) 2002-2003, Yannick Verschueren + * Copyright (c) 2001-2003, David Janssens + * Copyright (c) 2011-2012, Centre National d'Etudes Spatiales (CNES), France + * Copyright (c) 2012, CS Systemes d'Information, France + * + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS `AS IS' + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + + +---- + +RAQM + +The MIT License (MIT) + +Copyright © 2015 Information Technology Authority (ITA) +Copyright © 2016 Khaled Hosny + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +---- + +XAU + +Copyright 1988, 1993, 1994, 1998 The Open Group + +Permission to use, copy, modify, distribute, and sell this software and its +documentation for any purpose is hereby granted without fee, provided that +the above copyright notice appear in all copies and that both that +copyright notice and this permission notice appear in supporting +documentation. + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +OPEN GROUP BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of The Open Group shall not be +used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization from The Open Group. + + +---- + +XCB + +Copyright (C) 2001-2006 Bart Massey, Jamey Sharp, and Josh Triplett. +All Rights Reserved. + +Permission is hereby granted, free of charge, to any person +obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall +be included in all copies or substantial portions of the +Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR +PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS +BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the names of the authors +or their institutions shall not be used in advertising or +otherwise to promote the sale, use or other dealings in this +Software without prior written authorization from the +authors. + + +---- + +XDMCP + +Copyright 1989, 1998 The Open Group + +Permission to use, copy, modify, distribute, and sell this software and its +documentation for any purpose is hereby granted without fee, provided that +the above copyright notice appear in all copies and that both that +copyright notice and this permission notice appear in supporting +documentation. + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +OPEN GROUP BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN +AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +Except as contained in this notice, the name of The Open Group shall not be +used in advertising or otherwise to promote the sale, use or other dealings +in this Software without prior written authorization from The Open Group. + +Author: Keith Packard, MIT X Consortium + + +---- + +ZLIB + + (C) 1995-2017 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +If you use the zlib library in a product, we would appreciate *not* receiving +lengthy legal documents to sign. The sources are provided for free but without +warranty of any kind. The library has been entirely written by Jean-loup +Gailly and Mark Adler; it does not include third-party code. + +If you redistribute modified sources, we would appreciate that you include in +the file ChangeLog history information documenting your changes. Please read +the FAQ for more information on the distribution of modified source versions. + + +---- + +ZSTD + +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/sboms/auditwheel.cdx.json b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/sboms/auditwheel.cdx.json new file mode 100644 index 0000000000000000000000000000000000000000..2ecab518f15b3869512c70519cc579c4f9f29974 --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/sboms/auditwheel.cdx.json @@ -0,0 +1 @@ +{"bomFormat": "CycloneDX", "specVersion": "1.4", "version": 1, "metadata": {"component": {"type": "library", "bom-ref": "pkg:pypi/pillow@12.1.0?file_name=pillow-12.1.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", "name": "pillow", "version": "12.1.0", "purl": "pkg:pypi/pillow@12.1.0?file_name=pillow-12.1.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl"}, "tools": [{"name": "auditwheel", "version": "6.5.0"}]}, "components": [{"type": "library", "bom-ref": "pkg:pypi/pillow@12.1.0?file_name=pillow-12.1.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", "name": "pillow", "version": "12.1.0", "purl": "pkg:pypi/pillow@12.1.0?file_name=pillow-12.1.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl"}, {"type": "library", "bom-ref": "pkg:rpm/almalinux/libXau@1.0.9-3.el8#ac77887fa9a50833ff7a34d7e27ebe243468552bc94f6628693cb1dfdaf102e7", "name": "libXau", "version": "1.0.9-3.el8", "purl": "pkg:rpm/almalinux/libXau@1.0.9-3.el8"}], "dependencies": [{"ref": "pkg:pypi/pillow@12.1.0?file_name=pillow-12.1.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", "dependsOn": ["pkg:rpm/almalinux/libXau@1.0.9-3.el8#ac77887fa9a50833ff7a34d7e27ebe243468552bc94f6628693cb1dfdaf102e7"]}, {"ref": "pkg:rpm/almalinux/libXau@1.0.9-3.el8#ac77887fa9a50833ff7a34d7e27ebe243468552bc94f6628693cb1dfdaf102e7"}]} \ No newline at end of file diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/top_level.txt b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/top_level.txt new file mode 100644 index 0000000000000000000000000000000000000000..b338169ce0c740c335bfe82912227ae8637bd492 --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/top_level.txt @@ -0,0 +1 @@ +PIL diff --git a/lib/python3.12/site-packages/pillow-12.1.0.dist-info/zip-safe b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/zip-safe new file mode 100644 index 0000000000000000000000000000000000000000..8b137891791fe96927ad78e64b0aad7bded08bdc --- /dev/null +++ b/lib/python3.12/site-packages/pillow-12.1.0.dist-info/zip-safe @@ -0,0 +1 @@ + diff --git a/lib/python3.12/site-packages/psutil/__init__.py b/lib/python3.12/site-packages/psutil/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..cf4a58057f9cd86df47159f5778cff467a391e36 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/__init__.py @@ -0,0 +1,2407 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""psutil is a cross-platform library for retrieving information on +running processes and system utilization (CPU, memory, disks, network, +sensors) in Python. Supported platforms: + + - Linux + - Windows + - macOS + - FreeBSD + - OpenBSD + - NetBSD + - Sun Solaris + - AIX + +Supported Python versions are cPython 3.6+ and PyPy. +""" + +import collections +import contextlib +import datetime +import functools +import os +import signal +import socket +import subprocess +import sys +import threading +import time + + +try: + import pwd +except ImportError: + pwd = None + +from . import _common +from ._common import AIX +from ._common import BSD +from ._common import CONN_CLOSE +from ._common import CONN_CLOSE_WAIT +from ._common import CONN_CLOSING +from ._common import CONN_ESTABLISHED +from ._common import CONN_FIN_WAIT1 +from ._common import CONN_FIN_WAIT2 +from ._common import CONN_LAST_ACK +from ._common import CONN_LISTEN +from ._common import CONN_NONE +from ._common import CONN_SYN_RECV +from ._common import CONN_SYN_SENT +from ._common import CONN_TIME_WAIT +from ._common import FREEBSD +from ._common import LINUX +from ._common import MACOS +from ._common import NETBSD +from ._common import NIC_DUPLEX_FULL +from ._common import NIC_DUPLEX_HALF +from ._common import NIC_DUPLEX_UNKNOWN +from ._common import OPENBSD +from ._common import OSX # deprecated alias +from ._common import POSIX +from ._common import POWER_TIME_UNKNOWN +from ._common import POWER_TIME_UNLIMITED +from ._common import STATUS_DEAD +from ._common import STATUS_DISK_SLEEP +from ._common import STATUS_IDLE +from ._common import STATUS_LOCKED +from ._common import STATUS_PARKED +from ._common import STATUS_RUNNING +from ._common import STATUS_SLEEPING +from ._common import STATUS_STOPPED +from ._common import STATUS_TRACING_STOP +from ._common import STATUS_WAITING +from ._common import STATUS_WAKING +from ._common import STATUS_ZOMBIE +from ._common import SUNOS +from ._common import WINDOWS +from ._common import AccessDenied +from ._common import Error +from ._common import NoSuchProcess +from ._common import TimeoutExpired +from ._common import ZombieProcess +from ._common import debug +from ._common import memoize_when_activated +from ._common import wrap_numbers as _wrap_numbers + + +if LINUX: + # This is public API and it will be retrieved from _pslinux.py + # via sys.modules. + PROCFS_PATH = "/proc" + + from . import _pslinux as _psplatform + from ._pslinux import IOPRIO_CLASS_BE # noqa: F401 + from ._pslinux import IOPRIO_CLASS_IDLE # noqa: F401 + from ._pslinux import IOPRIO_CLASS_NONE # noqa: F401 + from ._pslinux import IOPRIO_CLASS_RT # noqa: F401 + +elif WINDOWS: + from . import _pswindows as _psplatform + from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import HIGH_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import IDLE_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import NORMAL_PRIORITY_CLASS # noqa: F401 + from ._psutil_windows import REALTIME_PRIORITY_CLASS # noqa: F401 + from ._pswindows import CONN_DELETE_TCB # noqa: F401 + from ._pswindows import IOPRIO_HIGH # noqa: F401 + from ._pswindows import IOPRIO_LOW # noqa: F401 + from ._pswindows import IOPRIO_NORMAL # noqa: F401 + from ._pswindows import IOPRIO_VERYLOW # noqa: F401 + +elif MACOS: + from . import _psosx as _psplatform + +elif BSD: + from . import _psbsd as _psplatform + +elif SUNOS: + from . import _pssunos as _psplatform + from ._pssunos import CONN_BOUND # noqa: F401 + from ._pssunos import CONN_IDLE # noqa: F401 + + # This is public writable API which is read from _pslinux.py and + # _pssunos.py via sys.modules. + PROCFS_PATH = "/proc" + +elif AIX: + from . import _psaix as _psplatform + + # This is public API and it will be retrieved from _pslinux.py + # via sys.modules. + PROCFS_PATH = "/proc" + +else: # pragma: no cover + msg = f"platform {sys.platform} is not supported" + raise NotImplementedError(msg) + + +# fmt: off +__all__ = [ + # exceptions + "Error", "NoSuchProcess", "ZombieProcess", "AccessDenied", + "TimeoutExpired", + + # constants + "version_info", "__version__", + + "STATUS_RUNNING", "STATUS_IDLE", "STATUS_SLEEPING", "STATUS_DISK_SLEEP", + "STATUS_STOPPED", "STATUS_TRACING_STOP", "STATUS_ZOMBIE", "STATUS_DEAD", + "STATUS_WAKING", "STATUS_LOCKED", "STATUS_WAITING", "STATUS_LOCKED", + "STATUS_PARKED", + + "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1", + "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT", + "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING", "CONN_NONE", + # "CONN_IDLE", "CONN_BOUND", + + "AF_LINK", + + "NIC_DUPLEX_FULL", "NIC_DUPLEX_HALF", "NIC_DUPLEX_UNKNOWN", + + "POWER_TIME_UNKNOWN", "POWER_TIME_UNLIMITED", + + "BSD", "FREEBSD", "LINUX", "NETBSD", "OPENBSD", "MACOS", "OSX", "POSIX", + "SUNOS", "WINDOWS", "AIX", + + # "RLIM_INFINITY", "RLIMIT_AS", "RLIMIT_CORE", "RLIMIT_CPU", "RLIMIT_DATA", + # "RLIMIT_FSIZE", "RLIMIT_LOCKS", "RLIMIT_MEMLOCK", "RLIMIT_NOFILE", + # "RLIMIT_NPROC", "RLIMIT_RSS", "RLIMIT_STACK", "RLIMIT_MSGQUEUE", + # "RLIMIT_NICE", "RLIMIT_RTPRIO", "RLIMIT_RTTIME", "RLIMIT_SIGPENDING", + + # classes + "Process", "Popen", + + # functions + "pid_exists", "pids", "process_iter", "wait_procs", # proc + "virtual_memory", "swap_memory", # memory + "cpu_times", "cpu_percent", "cpu_times_percent", "cpu_count", # cpu + "cpu_stats", # "cpu_freq", "getloadavg" + "net_io_counters", "net_connections", "net_if_addrs", # network + "net_if_stats", + "disk_io_counters", "disk_partitions", "disk_usage", # disk + # "sensors_temperatures", "sensors_battery", "sensors_fans" # sensors + "users", "boot_time", # others +] +# fmt: on + + +__all__.extend(_psplatform.__extra__all__) + +# Linux, FreeBSD +if hasattr(_psplatform.Process, "rlimit"): + # Populate global namespace with RLIM* constants. + from . import _psutil_posix + + _globals = globals() + _name = None + for _name in dir(_psutil_posix): + if _name.startswith('RLIM') and _name.isupper(): + _globals[_name] = getattr(_psutil_posix, _name) + __all__.append(_name) + del _globals, _name + +AF_LINK = _psplatform.AF_LINK + +__author__ = "Giampaolo Rodola'" +__version__ = "7.0.0" +version_info = tuple(int(num) for num in __version__.split('.')) + +_timer = getattr(time, 'monotonic', time.time) +_TOTAL_PHYMEM = None +_LOWEST_PID = None +_SENTINEL = object() + +# Sanity check in case the user messed up with psutil installation +# or did something weird with sys.path. In this case we might end +# up importing a python module using a C extension module which +# was compiled for a different version of psutil. +# We want to prevent that by failing sooner rather than later. +# See: https://github.com/giampaolo/psutil/issues/564 +if int(__version__.replace('.', '')) != getattr( + _psplatform.cext, 'version', None +): + msg = f"version conflict: {_psplatform.cext.__file__!r} C extension " + msg += "module was built for another version of psutil" + if hasattr(_psplatform.cext, 'version'): + v = ".".join(list(str(_psplatform.cext.version))) + msg += f" ({v} instead of {__version__})" + else: + msg += f" (different than {__version__})" + what = getattr( + _psplatform.cext, + "__file__", + "the existing psutil install directory", + ) + msg += f"; you may try to 'pip uninstall psutil', manually remove {what}" + msg += " or clean the virtual env somehow, then reinstall" + raise ImportError(msg) + + +# ===================================================================== +# --- Utils +# ===================================================================== + + +if hasattr(_psplatform, 'ppid_map'): + # Faster version (Windows and Linux). + _ppid_map = _psplatform.ppid_map +else: # pragma: no cover + + def _ppid_map(): + """Return a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + for pid in pids(): + try: + ret[pid] = _psplatform.Process(pid).ppid() + except (NoSuchProcess, ZombieProcess): + pass + return ret + + +def _pprint_secs(secs): + """Format seconds in a human readable form.""" + now = time.time() + secs_ago = int(now - secs) + fmt = "%H:%M:%S" if secs_ago < 60 * 60 * 24 else "%Y-%m-%d %H:%M:%S" + return datetime.datetime.fromtimestamp(secs).strftime(fmt) + + +def _check_conn_kind(kind): + """Check net_connections()'s `kind` parameter.""" + kinds = tuple(_common.conn_tmap) + if kind not in kinds: + msg = f"invalid kind argument {kind!r}; valid ones are: {kinds}" + raise ValueError(msg) + + +# ===================================================================== +# --- Process class +# ===================================================================== + + +class Process: + """Represents an OS process with the given PID. + If PID is omitted current process PID (os.getpid()) is used. + Raise NoSuchProcess if PID does not exist. + + Note that most of the methods of this class do not make sure that + the PID of the process being queried has been reused. That means + that you may end up retrieving information for another process. + + The only exceptions for which process identity is pre-emptively + checked and guaranteed are: + + - parent() + - children() + - nice() (set) + - ionice() (set) + - rlimit() (set) + - cpu_affinity (set) + - suspend() + - resume() + - send_signal() + - terminate() + - kill() + + To prevent this problem for all other methods you can use + is_running() before querying the process. + """ + + def __init__(self, pid=None): + self._init(pid) + + def _init(self, pid, _ignore_nsp=False): + if pid is None: + pid = os.getpid() + else: + if pid < 0: + msg = f"pid must be a positive integer (got {pid})" + raise ValueError(msg) + try: + _psplatform.cext.check_pid_range(pid) + except OverflowError as err: + msg = "process PID out of range" + raise NoSuchProcess(pid, msg=msg) from err + + self._pid = pid + self._name = None + self._exe = None + self._create_time = None + self._gone = False + self._pid_reused = False + self._hash = None + self._lock = threading.RLock() + # used for caching on Windows only (on POSIX ppid may change) + self._ppid = None + # platform-specific modules define an _psplatform.Process + # implementation class + self._proc = _psplatform.Process(pid) + self._last_sys_cpu_times = None + self._last_proc_cpu_times = None + self._exitcode = _SENTINEL + self._ident = (self.pid, None) + try: + self._ident = self._get_ident() + except AccessDenied: + # This should happen on Windows only, since we use the fast + # create time method. AFAIK, on all other platforms we are + # able to get create time for all PIDs. + pass + except ZombieProcess: + # Zombies can still be queried by this class (although + # not always) and pids() return them so just go on. + pass + except NoSuchProcess: + if not _ignore_nsp: + msg = "process PID not found" + raise NoSuchProcess(pid, msg=msg) from None + self._gone = True + + def _get_ident(self): + """Return a (pid, uid) tuple which is supposed to identify a + Process instance univocally over time. The PID alone is not + enough, as it can be assigned to a new process after this one + terminates, so we add process creation time to the mix. We need + this in order to prevent killing the wrong process later on. + This is also known as PID reuse or PID recycling problem. + + The reliability of this strategy mostly depends on + create_time() precision, which is 0.01 secs on Linux. The + assumption is that, after a process terminates, the kernel + won't reuse the same PID after such a short period of time + (0.01 secs). Technically this is inherently racy, but + practically it should be good enough. + """ + if WINDOWS: + # Use create_time() fast method in order to speedup + # `process_iter()`. This means we'll get AccessDenied for + # most ADMIN processes, but that's fine since it means + # we'll also get AccessDenied on kill(). + # https://github.com/giampaolo/psutil/issues/2366#issuecomment-2381646555 + self._create_time = self._proc.create_time(fast_only=True) + return (self.pid, self._create_time) + else: + return (self.pid, self.create_time()) + + def __str__(self): + info = collections.OrderedDict() + info["pid"] = self.pid + if self._name: + info['name'] = self._name + with self.oneshot(): + if self._pid_reused: + info["status"] = "terminated + PID reused" + else: + try: + info["name"] = self.name() + info["status"] = self.status() + except ZombieProcess: + info["status"] = "zombie" + except NoSuchProcess: + info["status"] = "terminated" + except AccessDenied: + pass + + if self._exitcode not in {_SENTINEL, None}: + info["exitcode"] = self._exitcode + if self._create_time is not None: + info['started'] = _pprint_secs(self._create_time) + + return "{}.{}({})".format( + self.__class__.__module__, + self.__class__.__name__, + ", ".join([f"{k}={v!r}" for k, v in info.items()]), + ) + + __repr__ = __str__ + + def __eq__(self, other): + # Test for equality with another Process object based + # on PID and creation time. + if not isinstance(other, Process): + return NotImplemented + if OPENBSD or NETBSD: # pragma: no cover + # Zombie processes on Open/NetBSD have a creation time of + # 0.0. This covers the case when a process started normally + # (so it has a ctime), then it turned into a zombie. It's + # important to do this because is_running() depends on + # __eq__. + pid1, ident1 = self._ident + pid2, ident2 = other._ident + if pid1 == pid2: + if ident1 and not ident2: + try: + return self.status() == STATUS_ZOMBIE + except Error: + pass + return self._ident == other._ident + + def __ne__(self, other): + return not self == other + + def __hash__(self): + if self._hash is None: + self._hash = hash(self._ident) + return self._hash + + def _raise_if_pid_reused(self): + """Raises NoSuchProcess in case process PID has been reused.""" + if self._pid_reused or (not self.is_running() and self._pid_reused): + # We may directly raise NSP in here already if PID is just + # not running, but I prefer NSP to be raised naturally by + # the actual Process API call. This way unit tests will tell + # us if the API is broken (aka don't raise NSP when it + # should). We also remain consistent with all other "get" + # APIs which don't use _raise_if_pid_reused(). + msg = "process no longer exists and its PID has been reused" + raise NoSuchProcess(self.pid, self._name, msg=msg) + + @property + def pid(self): + """The process PID.""" + return self._pid + + # --- utility methods + + @contextlib.contextmanager + def oneshot(self): + """Utility context manager which considerably speeds up the + retrieval of multiple process information at the same time. + + Internally different process info (e.g. name, ppid, uids, + gids, ...) may be fetched by using the same routine, but + only one information is returned and the others are discarded. + When using this context manager the internal routine is + executed once (in the example below on name()) and the + other info are cached. + + The cache is cleared when exiting the context manager block. + The advice is to use this every time you retrieve more than + one information about the process. If you're lucky, you'll + get a hell of a speedup. + + >>> import psutil + >>> p = psutil.Process() + >>> with p.oneshot(): + ... p.name() # collect multiple info + ... p.cpu_times() # return cached value + ... p.cpu_percent() # return cached value + ... p.create_time() # return cached value + ... + >>> + """ + with self._lock: + if hasattr(self, "_cache"): + # NOOP: this covers the use case where the user enters the + # context twice: + # + # >>> with p.oneshot(): + # ... with p.oneshot(): + # ... + # + # Also, since as_dict() internally uses oneshot() + # I expect that the code below will be a pretty common + # "mistake" that the user will make, so let's guard + # against that: + # + # >>> with p.oneshot(): + # ... p.as_dict() + # ... + yield + else: + try: + # cached in case cpu_percent() is used + self.cpu_times.cache_activate(self) + # cached in case memory_percent() is used + self.memory_info.cache_activate(self) + # cached in case parent() is used + self.ppid.cache_activate(self) + # cached in case username() is used + if POSIX: + self.uids.cache_activate(self) + # specific implementation cache + self._proc.oneshot_enter() + yield + finally: + self.cpu_times.cache_deactivate(self) + self.memory_info.cache_deactivate(self) + self.ppid.cache_deactivate(self) + if POSIX: + self.uids.cache_deactivate(self) + self._proc.oneshot_exit() + + def as_dict(self, attrs=None, ad_value=None): + """Utility method returning process information as a + hashable dictionary. + If *attrs* is specified it must be a list of strings + reflecting available Process class' attribute names + (e.g. ['cpu_times', 'name']) else all public (read + only) attributes are assumed. + *ad_value* is the value which gets assigned in case + AccessDenied or ZombieProcess exception is raised when + retrieving that particular process information. + """ + valid_names = _as_dict_attrnames + if attrs is not None: + if not isinstance(attrs, (list, tuple, set, frozenset)): + msg = f"invalid attrs type {type(attrs)}" + raise TypeError(msg) + attrs = set(attrs) + invalid_names = attrs - valid_names + if invalid_names: + msg = "invalid attr name{} {}".format( + "s" if len(invalid_names) > 1 else "", + ", ".join(map(repr, invalid_names)), + ) + raise ValueError(msg) + + retdict = {} + ls = attrs or valid_names + with self.oneshot(): + for name in ls: + try: + if name == 'pid': + ret = self.pid + else: + meth = getattr(self, name) + ret = meth() + except (AccessDenied, ZombieProcess): + ret = ad_value + except NotImplementedError: + # in case of not implemented functionality (may happen + # on old or exotic systems) we want to crash only if + # the user explicitly asked for that particular attr + if attrs: + raise + continue + retdict[name] = ret + return retdict + + def parent(self): + """Return the parent process as a Process object pre-emptively + checking whether PID has been reused. + If no parent is known return None. + """ + lowest_pid = _LOWEST_PID if _LOWEST_PID is not None else pids()[0] + if self.pid == lowest_pid: + return None + ppid = self.ppid() + if ppid is not None: + ctime = self.create_time() + try: + parent = Process(ppid) + if parent.create_time() <= ctime: + return parent + # ...else ppid has been reused by another process + except NoSuchProcess: + pass + + def parents(self): + """Return the parents of this process as a list of Process + instances. If no parents are known return an empty list. + """ + parents = [] + proc = self.parent() + while proc is not None: + parents.append(proc) + proc = proc.parent() + return parents + + def is_running(self): + """Return whether this process is running. + + It also checks if PID has been reused by another process, in + which case it will remove the process from `process_iter()` + internal cache and return False. + """ + if self._gone or self._pid_reused: + return False + try: + # Checking if PID is alive is not enough as the PID might + # have been reused by another process. Process identity / + # uniqueness over time is guaranteed by (PID + creation + # time) and that is verified in __eq__. + self._pid_reused = self != Process(self.pid) + if self._pid_reused: + _pids_reused.add(self.pid) + raise NoSuchProcess(self.pid) + return True + except ZombieProcess: + # We should never get here as it's already handled in + # Process.__init__; here just for extra safety. + return True + except NoSuchProcess: + self._gone = True + return False + + # --- actual API + + @memoize_when_activated + def ppid(self): + """The process parent PID. + On Windows the return value is cached after first call. + """ + # On POSIX we don't want to cache the ppid as it may unexpectedly + # change to 1 (init) in case this process turns into a zombie: + # https://github.com/giampaolo/psutil/issues/321 + # http://stackoverflow.com/questions/356722/ + + # XXX should we check creation time here rather than in + # Process.parent()? + self._raise_if_pid_reused() + if POSIX: + return self._proc.ppid() + else: # pragma: no cover + self._ppid = self._ppid or self._proc.ppid() + return self._ppid + + def name(self): + """The process name. The return value is cached after first call.""" + # Process name is only cached on Windows as on POSIX it may + # change, see: + # https://github.com/giampaolo/psutil/issues/692 + if WINDOWS and self._name is not None: + return self._name + name = self._proc.name() + if POSIX and len(name) >= 15: + # On UNIX the name gets truncated to the first 15 characters. + # If it matches the first part of the cmdline we return that + # one instead because it's usually more explicative. + # Examples are "gnome-keyring-d" vs. "gnome-keyring-daemon". + try: + cmdline = self.cmdline() + except (AccessDenied, ZombieProcess): + # Just pass and return the truncated name: it's better + # than nothing. Note: there are actual cases where a + # zombie process can return a name() but not a + # cmdline(), see: + # https://github.com/giampaolo/psutil/issues/2239 + pass + else: + if cmdline: + extended_name = os.path.basename(cmdline[0]) + if extended_name.startswith(name): + name = extended_name + self._name = name + self._proc._name = name + return name + + def exe(self): + """The process executable as an absolute path. + May also be an empty string. + The return value is cached after first call. + """ + + def guess_it(fallback): + # try to guess exe from cmdline[0] in absence of a native + # exe representation + cmdline = self.cmdline() + if cmdline and hasattr(os, 'access') and hasattr(os, 'X_OK'): + exe = cmdline[0] # the possible exe + # Attempt to guess only in case of an absolute path. + # It is not safe otherwise as the process might have + # changed cwd. + if ( + os.path.isabs(exe) + and os.path.isfile(exe) + and os.access(exe, os.X_OK) + ): + return exe + if isinstance(fallback, AccessDenied): + raise fallback + return fallback + + if self._exe is None: + try: + exe = self._proc.exe() + except AccessDenied as err: + return guess_it(fallback=err) + else: + if not exe: + # underlying implementation can legitimately return an + # empty string; if that's the case we don't want to + # raise AD while guessing from the cmdline + try: + exe = guess_it(fallback=exe) + except AccessDenied: + pass + self._exe = exe + return self._exe + + def cmdline(self): + """The command line this process has been called with.""" + return self._proc.cmdline() + + def status(self): + """The process current status as a STATUS_* constant.""" + try: + return self._proc.status() + except ZombieProcess: + return STATUS_ZOMBIE + + def username(self): + """The name of the user that owns the process. + On UNIX this is calculated by using *real* process uid. + """ + if POSIX: + if pwd is None: + # might happen if python was installed from sources + msg = "requires pwd module shipped with standard python" + raise ImportError(msg) + real_uid = self.uids().real + try: + return pwd.getpwuid(real_uid).pw_name + except KeyError: + # the uid can't be resolved by the system + return str(real_uid) + else: + return self._proc.username() + + def create_time(self): + """The process creation time as a floating point number + expressed in seconds since the epoch. + The return value is cached after first call. + """ + if self._create_time is None: + self._create_time = self._proc.create_time() + return self._create_time + + def cwd(self): + """Process current working directory as an absolute path.""" + return self._proc.cwd() + + def nice(self, value=None): + """Get or set process niceness (priority).""" + if value is None: + return self._proc.nice_get() + else: + self._raise_if_pid_reused() + self._proc.nice_set(value) + + if POSIX: + + @memoize_when_activated + def uids(self): + """Return process UIDs as a (real, effective, saved) + namedtuple. + """ + return self._proc.uids() + + def gids(self): + """Return process GIDs as a (real, effective, saved) + namedtuple. + """ + return self._proc.gids() + + def terminal(self): + """The terminal associated with this process, if any, + else None. + """ + return self._proc.terminal() + + def num_fds(self): + """Return the number of file descriptors opened by this + process (POSIX only). + """ + return self._proc.num_fds() + + # Linux, BSD, AIX and Windows only + if hasattr(_psplatform.Process, "io_counters"): + + def io_counters(self): + """Return process I/O statistics as a + (read_count, write_count, read_bytes, write_bytes) + namedtuple. + Those are the number of read/write calls performed and the + amount of bytes read and written by the process. + """ + return self._proc.io_counters() + + # Linux and Windows + if hasattr(_psplatform.Process, "ionice_get"): + + def ionice(self, ioclass=None, value=None): + """Get or set process I/O niceness (priority). + + On Linux *ioclass* is one of the IOPRIO_CLASS_* constants. + *value* is a number which goes from 0 to 7. The higher the + value, the lower the I/O priority of the process. + + On Windows only *ioclass* is used and it can be set to 2 + (normal), 1 (low) or 0 (very low). + + Available on Linux and Windows > Vista only. + """ + if ioclass is None: + if value is not None: + msg = "'ioclass' argument must be specified" + raise ValueError(msg) + return self._proc.ionice_get() + else: + self._raise_if_pid_reused() + return self._proc.ionice_set(ioclass, value) + + # Linux / FreeBSD only + if hasattr(_psplatform.Process, "rlimit"): + + def rlimit(self, resource, limits=None): + """Get or set process resource limits as a (soft, hard) + tuple. + + *resource* is one of the RLIMIT_* constants. + *limits* is supposed to be a (soft, hard) tuple. + + See "man prlimit" for further info. + Available on Linux and FreeBSD only. + """ + if limits is not None: + self._raise_if_pid_reused() + return self._proc.rlimit(resource, limits) + + # Windows, Linux and FreeBSD only + if hasattr(_psplatform.Process, "cpu_affinity_get"): + + def cpu_affinity(self, cpus=None): + """Get or set process CPU affinity. + If specified, *cpus* must be a list of CPUs for which you + want to set the affinity (e.g. [0, 1]). + If an empty list is passed, all egible CPUs are assumed + (and set). + (Windows, Linux and BSD only). + """ + if cpus is None: + return sorted(set(self._proc.cpu_affinity_get())) + else: + self._raise_if_pid_reused() + if not cpus: + if hasattr(self._proc, "_get_eligible_cpus"): + cpus = self._proc._get_eligible_cpus() + else: + cpus = tuple(range(len(cpu_times(percpu=True)))) + self._proc.cpu_affinity_set(list(set(cpus))) + + # Linux, FreeBSD, SunOS + if hasattr(_psplatform.Process, "cpu_num"): + + def cpu_num(self): + """Return what CPU this process is currently running on. + The returned number should be <= psutil.cpu_count() + and <= len(psutil.cpu_percent(percpu=True)). + It may be used in conjunction with + psutil.cpu_percent(percpu=True) to observe the system + workload distributed across CPUs. + """ + return self._proc.cpu_num() + + # All platforms has it, but maybe not in the future. + if hasattr(_psplatform.Process, "environ"): + + def environ(self): + """The environment variables of the process as a dict. Note: this + might not reflect changes made after the process started. + """ + return self._proc.environ() + + if WINDOWS: + + def num_handles(self): + """Return the number of handles opened by this process + (Windows only). + """ + return self._proc.num_handles() + + def num_ctx_switches(self): + """Return the number of voluntary and involuntary context + switches performed by this process. + """ + return self._proc.num_ctx_switches() + + def num_threads(self): + """Return the number of threads used by this process.""" + return self._proc.num_threads() + + if hasattr(_psplatform.Process, "threads"): + + def threads(self): + """Return threads opened by process as a list of + (id, user_time, system_time) namedtuples representing + thread id and thread CPU times (user/system). + On OpenBSD this method requires root access. + """ + return self._proc.threads() + + def children(self, recursive=False): + """Return the children of this process as a list of Process + instances, pre-emptively checking whether PID has been reused. + If *recursive* is True return all the parent descendants. + + Example (A == this process): + + A ─┐ + │ + ├─ B (child) ─┐ + │ └─ X (grandchild) ─┐ + │ └─ Y (great grandchild) + ├─ C (child) + └─ D (child) + + >>> import psutil + >>> p = psutil.Process() + >>> p.children() + B, C, D + >>> p.children(recursive=True) + B, X, Y, C, D + + Note that in the example above if process X disappears + process Y won't be listed as the reference to process A + is lost. + """ + self._raise_if_pid_reused() + ppid_map = _ppid_map() + ret = [] + if not recursive: + for pid, ppid in ppid_map.items(): + if ppid == self.pid: + try: + child = Process(pid) + # if child happens to be older than its parent + # (self) it means child's PID has been reused + if self.create_time() <= child.create_time(): + ret.append(child) + except (NoSuchProcess, ZombieProcess): + pass + else: + # Construct a {pid: [child pids]} dict + reverse_ppid_map = collections.defaultdict(list) + for pid, ppid in ppid_map.items(): + reverse_ppid_map[ppid].append(pid) + # Recursively traverse that dict, starting from self.pid, + # such that we only call Process() on actual children + seen = set() + stack = [self.pid] + while stack: + pid = stack.pop() + if pid in seen: + # Since pids can be reused while the ppid_map is + # constructed, there may be rare instances where + # there's a cycle in the recorded process "tree". + continue + seen.add(pid) + for child_pid in reverse_ppid_map[pid]: + try: + child = Process(child_pid) + # if child happens to be older than its parent + # (self) it means child's PID has been reused + intime = self.create_time() <= child.create_time() + if intime: + ret.append(child) + stack.append(child_pid) + except (NoSuchProcess, ZombieProcess): + pass + return ret + + def cpu_percent(self, interval=None): + """Return a float representing the current process CPU + utilization as a percentage. + + When *interval* is 0.0 or None (default) compares process times + to system CPU times elapsed since last call, returning + immediately (non-blocking). That means that the first time + this is called it will return a meaningful 0.0 value. + + When *interval* is > 0.0 compares process times to system CPU + times elapsed before and after the interval (blocking). + + In this case is recommended for accuracy that this function + be called with at least 0.1 seconds between calls. + + A value > 100.0 can be returned in case of processes running + multiple threads on different CPU cores. + + The returned value is explicitly NOT split evenly between + all available logical CPUs. This means that a busy loop process + running on a system with 2 logical CPUs will be reported as + having 100% CPU utilization instead of 50%. + + Examples: + + >>> import psutil + >>> p = psutil.Process(os.getpid()) + >>> # blocking + >>> p.cpu_percent(interval=1) + 2.0 + >>> # non-blocking (percentage since last call) + >>> p.cpu_percent(interval=None) + 2.9 + >>> + """ + blocking = interval is not None and interval > 0.0 + if interval is not None and interval < 0: + msg = f"interval is not positive (got {interval!r})" + raise ValueError(msg) + num_cpus = cpu_count() or 1 + + def timer(): + return _timer() * num_cpus + + if blocking: + st1 = timer() + pt1 = self._proc.cpu_times() + time.sleep(interval) + st2 = timer() + pt2 = self._proc.cpu_times() + else: + st1 = self._last_sys_cpu_times + pt1 = self._last_proc_cpu_times + st2 = timer() + pt2 = self._proc.cpu_times() + if st1 is None or pt1 is None: + self._last_sys_cpu_times = st2 + self._last_proc_cpu_times = pt2 + return 0.0 + + delta_proc = (pt2.user - pt1.user) + (pt2.system - pt1.system) + delta_time = st2 - st1 + # reset values for next call in case of interval == None + self._last_sys_cpu_times = st2 + self._last_proc_cpu_times = pt2 + + try: + # This is the utilization split evenly between all CPUs. + # E.g. a busy loop process on a 2-CPU-cores system at this + # point is reported as 50% instead of 100%. + overall_cpus_percent = (delta_proc / delta_time) * 100 + except ZeroDivisionError: + # interval was too low + return 0.0 + else: + # Note 1: + # in order to emulate "top" we multiply the value for the num + # of CPU cores. This way the busy process will be reported as + # having 100% (or more) usage. + # + # Note 2: + # taskmgr.exe on Windows differs in that it will show 50% + # instead. + # + # Note 3: + # a percentage > 100 is legitimate as it can result from a + # process with multiple threads running on different CPU + # cores (top does the same), see: + # http://stackoverflow.com/questions/1032357 + # https://github.com/giampaolo/psutil/issues/474 + single_cpu_percent = overall_cpus_percent * num_cpus + return round(single_cpu_percent, 1) + + @memoize_when_activated + def cpu_times(self): + """Return a (user, system, children_user, children_system) + namedtuple representing the accumulated process time, in + seconds. + This is similar to os.times() but per-process. + On macOS and Windows children_user and children_system are + always set to 0. + """ + return self._proc.cpu_times() + + @memoize_when_activated + def memory_info(self): + """Return a namedtuple with variable fields depending on the + platform, representing memory information about the process. + + The "portable" fields available on all platforms are `rss` and `vms`. + + All numbers are expressed in bytes. + """ + return self._proc.memory_info() + + def memory_full_info(self): + """This method returns the same information as memory_info(), + plus, on some platform (Linux, macOS, Windows), also provides + additional metrics (USS, PSS and swap). + The additional metrics provide a better representation of actual + process memory usage. + + Namely USS is the memory which is unique to a process and which + would be freed if the process was terminated right now. + + It does so by passing through the whole process address. + As such it usually requires higher user privileges than + memory_info() and is considerably slower. + """ + return self._proc.memory_full_info() + + def memory_percent(self, memtype="rss"): + """Compare process memory to total physical system memory and + calculate process memory utilization as a percentage. + *memtype* argument is a string that dictates what type of + process memory you want to compare against (defaults to "rss"). + The list of available strings can be obtained like this: + + >>> psutil.Process().memory_info()._fields + ('rss', 'vms', 'shared', 'text', 'lib', 'data', 'dirty', 'uss', 'pss') + """ + valid_types = list(_psplatform.pfullmem._fields) + if memtype not in valid_types: + msg = ( + f"invalid memtype {memtype!r}; valid types are" + f" {tuple(valid_types)!r}" + ) + raise ValueError(msg) + fun = ( + self.memory_info + if memtype in _psplatform.pmem._fields + else self.memory_full_info + ) + metrics = fun() + value = getattr(metrics, memtype) + + # use cached value if available + total_phymem = _TOTAL_PHYMEM or virtual_memory().total + if not total_phymem > 0: + # we should never get here + msg = ( + "can't calculate process memory percent because total physical" + f" system memory is not positive ({total_phymem!r})" + ) + raise ValueError(msg) + return (value / float(total_phymem)) * 100 + + if hasattr(_psplatform.Process, "memory_maps"): + + def memory_maps(self, grouped=True): + """Return process' mapped memory regions as a list of namedtuples + whose fields are variable depending on the platform. + + If *grouped* is True the mapped regions with the same 'path' + are grouped together and the different memory fields are summed. + + If *grouped* is False every mapped region is shown as a single + entity and the namedtuple will also include the mapped region's + address space ('addr') and permission set ('perms'). + """ + it = self._proc.memory_maps() + if grouped: + d = {} + for tupl in it: + path = tupl[2] + nums = tupl[3:] + try: + d[path] = list(map(lambda x, y: x + y, d[path], nums)) + except KeyError: + d[path] = nums + nt = _psplatform.pmmap_grouped + return [nt(path, *d[path]) for path in d] + else: + nt = _psplatform.pmmap_ext + return [nt(*x) for x in it] + + def open_files(self): + """Return files opened by process as a list of + (path, fd) namedtuples including the absolute file name + and file descriptor number. + """ + return self._proc.open_files() + + def net_connections(self, kind='inet'): + """Return socket connections opened by process as a list of + (fd, family, type, laddr, raddr, status) namedtuples. + The *kind* parameter filters for connections that match the + following criteria: + + +------------+----------------------------------------------------+ + | Kind Value | Connections using | + +------------+----------------------------------------------------+ + | inet | IPv4 and IPv6 | + | inet4 | IPv4 | + | inet6 | IPv6 | + | tcp | TCP | + | tcp4 | TCP over IPv4 | + | tcp6 | TCP over IPv6 | + | udp | UDP | + | udp4 | UDP over IPv4 | + | udp6 | UDP over IPv6 | + | unix | UNIX socket (both UDP and TCP protocols) | + | all | the sum of all the possible families and protocols | + +------------+----------------------------------------------------+ + """ + _check_conn_kind(kind) + return self._proc.net_connections(kind) + + @_common.deprecated_method(replacement="net_connections") + def connections(self, kind="inet"): + return self.net_connections(kind=kind) + + # --- signals + + if POSIX: + + def _send_signal(self, sig): + assert not self.pid < 0, self.pid + self._raise_if_pid_reused() + + pid, ppid, name = self.pid, self._ppid, self._name + if pid == 0: + # see "man 2 kill" + msg = ( + "preventing sending signal to process with PID 0 as it " + "would affect every process in the process group of the " + "calling process (os.getpid()) instead of PID 0" + ) + raise ValueError(msg) + try: + os.kill(pid, sig) + except ProcessLookupError as err: + if OPENBSD and pid_exists(pid): + # We do this because os.kill() lies in case of + # zombie processes. + raise ZombieProcess(pid, name, ppid) from err + self._gone = True + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + + def send_signal(self, sig): + """Send a signal *sig* to process pre-emptively checking + whether PID has been reused (see signal module constants) . + On Windows only SIGTERM is valid and is treated as an alias + for kill(). + """ + if POSIX: + self._send_signal(sig) + else: # pragma: no cover + self._raise_if_pid_reused() + if sig != signal.SIGTERM and not self.is_running(): + msg = "process no longer exists" + raise NoSuchProcess(self.pid, self._name, msg=msg) + self._proc.send_signal(sig) + + def suspend(self): + """Suspend process execution with SIGSTOP pre-emptively checking + whether PID has been reused. + On Windows this has the effect of suspending all process threads. + """ + if POSIX: + self._send_signal(signal.SIGSTOP) + else: # pragma: no cover + self._raise_if_pid_reused() + self._proc.suspend() + + def resume(self): + """Resume process execution with SIGCONT pre-emptively checking + whether PID has been reused. + On Windows this has the effect of resuming all process threads. + """ + if POSIX: + self._send_signal(signal.SIGCONT) + else: # pragma: no cover + self._raise_if_pid_reused() + self._proc.resume() + + def terminate(self): + """Terminate the process with SIGTERM pre-emptively checking + whether PID has been reused. + On Windows this is an alias for kill(). + """ + if POSIX: + self._send_signal(signal.SIGTERM) + else: # pragma: no cover + self._raise_if_pid_reused() + self._proc.kill() + + def kill(self): + """Kill the current process with SIGKILL pre-emptively checking + whether PID has been reused. + """ + if POSIX: + self._send_signal(signal.SIGKILL) + else: # pragma: no cover + self._raise_if_pid_reused() + self._proc.kill() + + def wait(self, timeout=None): + """Wait for process to terminate and, if process is a children + of os.getpid(), also return its exit code, else None. + On Windows there's no such limitation (exit code is always + returned). + + If the process is already terminated immediately return None + instead of raising NoSuchProcess. + + If *timeout* (in seconds) is specified and process is still + alive raise TimeoutExpired. + + To wait for multiple Process(es) use psutil.wait_procs(). + """ + if timeout is not None and not timeout >= 0: + msg = "timeout must be a positive integer" + raise ValueError(msg) + if self._exitcode is not _SENTINEL: + return self._exitcode + self._exitcode = self._proc.wait(timeout) + return self._exitcode + + +# The valid attr names which can be processed by Process.as_dict(). +# fmt: off +_as_dict_attrnames = { + x for x in dir(Process) if not x.startswith("_") and x not in + {'send_signal', 'suspend', 'resume', 'terminate', 'kill', 'wait', + 'is_running', 'as_dict', 'parent', 'parents', 'children', 'rlimit', + 'connections', 'oneshot'} +} +# fmt: on + + +# ===================================================================== +# --- Popen class +# ===================================================================== + + +class Popen(Process): + """Same as subprocess.Popen, but in addition it provides all + psutil.Process methods in a single class. + For the following methods which are common to both classes, psutil + implementation takes precedence: + + * send_signal() + * terminate() + * kill() + + This is done in order to avoid killing another process in case its + PID has been reused, fixing BPO-6973. + + >>> import psutil + >>> from subprocess import PIPE + >>> p = psutil.Popen(["python", "-c", "print 'hi'"], stdout=PIPE) + >>> p.name() + 'python' + >>> p.uids() + user(real=1000, effective=1000, saved=1000) + >>> p.username() + 'giampaolo' + >>> p.communicate() + ('hi', None) + >>> p.terminate() + >>> p.wait(timeout=2) + 0 + >>> + """ + + def __init__(self, *args, **kwargs): + # Explicitly avoid to raise NoSuchProcess in case the process + # spawned by subprocess.Popen terminates too quickly, see: + # https://github.com/giampaolo/psutil/issues/193 + self.__subproc = subprocess.Popen(*args, **kwargs) + self._init(self.__subproc.pid, _ignore_nsp=True) + + def __dir__(self): + return sorted(set(dir(Popen) + dir(subprocess.Popen))) + + def __enter__(self): + if hasattr(self.__subproc, '__enter__'): + self.__subproc.__enter__() + return self + + def __exit__(self, *args, **kwargs): + if hasattr(self.__subproc, '__exit__'): + return self.__subproc.__exit__(*args, **kwargs) + else: + if self.stdout: + self.stdout.close() + if self.stderr: + self.stderr.close() + try: + # Flushing a BufferedWriter may raise an error. + if self.stdin: + self.stdin.close() + finally: + # Wait for the process to terminate, to avoid zombies. + self.wait() + + def __getattribute__(self, name): + try: + return object.__getattribute__(self, name) + except AttributeError: + try: + return object.__getattribute__(self.__subproc, name) + except AttributeError: + msg = f"{self.__class__!r} has no attribute {name!r}" + raise AttributeError(msg) from None + + def wait(self, timeout=None): + if self.__subproc.returncode is not None: + return self.__subproc.returncode + ret = super().wait(timeout) + self.__subproc.returncode = ret + return ret + + +# ===================================================================== +# --- system processes related functions +# ===================================================================== + + +def pids(): + """Return a list of current running PIDs.""" + global _LOWEST_PID + ret = sorted(_psplatform.pids()) + _LOWEST_PID = ret[0] + return ret + + +def pid_exists(pid): + """Return True if given PID exists in the current process list. + This is faster than doing "pid in psutil.pids()" and + should be preferred. + """ + if pid < 0: + return False + elif pid == 0 and POSIX: + # On POSIX we use os.kill() to determine PID existence. + # According to "man 2 kill" PID 0 has a special meaning + # though: it refers to <> and that is not we want + # to do here. + return pid in pids() + else: + return _psplatform.pid_exists(pid) + + +_pmap = {} +_pids_reused = set() + + +def process_iter(attrs=None, ad_value=None): + """Return a generator yielding a Process instance for all + running processes. + + Every new Process instance is only created once and then cached + into an internal table which is updated every time this is used. + Cache can optionally be cleared via `process_iter.clear_cache()`. + + The sorting order in which processes are yielded is based on + their PIDs. + + *attrs* and *ad_value* have the same meaning as in + Process.as_dict(). If *attrs* is specified as_dict() is called + and the resulting dict is stored as a 'info' attribute attached + to returned Process instance. + If *attrs* is an empty list it will retrieve all process info + (slow). + """ + global _pmap + + def add(pid): + proc = Process(pid) + pmap[proc.pid] = proc + return proc + + def remove(pid): + pmap.pop(pid, None) + + pmap = _pmap.copy() + a = set(pids()) + b = set(pmap.keys()) + new_pids = a - b + gone_pids = b - a + for pid in gone_pids: + remove(pid) + while _pids_reused: + pid = _pids_reused.pop() + debug(f"refreshing Process instance for reused PID {pid}") + remove(pid) + try: + ls = sorted(list(pmap.items()) + list(dict.fromkeys(new_pids).items())) + for pid, proc in ls: + try: + if proc is None: # new process + proc = add(pid) + if attrs is not None: + proc.info = proc.as_dict(attrs=attrs, ad_value=ad_value) + yield proc + except NoSuchProcess: + remove(pid) + finally: + _pmap = pmap + + +process_iter.cache_clear = lambda: _pmap.clear() # noqa: PLW0108 +process_iter.cache_clear.__doc__ = "Clear process_iter() internal cache." + + +def wait_procs(procs, timeout=None, callback=None): + """Convenience function which waits for a list of processes to + terminate. + + Return a (gone, alive) tuple indicating which processes + are gone and which ones are still alive. + + The gone ones will have a new *returncode* attribute indicating + process exit status (may be None). + + *callback* is a function which gets called every time a process + terminates (a Process instance is passed as callback argument). + + Function will return as soon as all processes terminate or when + *timeout* occurs. + Differently from Process.wait() it will not raise TimeoutExpired if + *timeout* occurs. + + Typical use case is: + + - send SIGTERM to a list of processes + - give them some time to terminate + - send SIGKILL to those ones which are still alive + + Example: + + >>> def on_terminate(proc): + ... print("process {} terminated".format(proc)) + ... + >>> for p in procs: + ... p.terminate() + ... + >>> gone, alive = wait_procs(procs, timeout=3, callback=on_terminate) + >>> for p in alive: + ... p.kill() + """ + + def check_gone(proc, timeout): + try: + returncode = proc.wait(timeout=timeout) + except (TimeoutExpired, subprocess.TimeoutExpired): + pass + else: + if returncode is not None or not proc.is_running(): + # Set new Process instance attribute. + proc.returncode = returncode + gone.add(proc) + if callback is not None: + callback(proc) + + if timeout is not None and not timeout >= 0: + msg = f"timeout must be a positive integer, got {timeout}" + raise ValueError(msg) + gone = set() + alive = set(procs) + if callback is not None and not callable(callback): + msg = f"callback {callback!r} is not a callable" + raise TypeError(msg) + if timeout is not None: + deadline = _timer() + timeout + + while alive: + if timeout is not None and timeout <= 0: + break + for proc in alive: + # Make sure that every complete iteration (all processes) + # will last max 1 sec. + # We do this because we don't want to wait too long on a + # single process: in case it terminates too late other + # processes may disappear in the meantime and their PID + # reused. + max_timeout = 1.0 / len(alive) + if timeout is not None: + timeout = min((deadline - _timer()), max_timeout) + if timeout <= 0: + break + check_gone(proc, timeout) + else: + check_gone(proc, max_timeout) + alive = alive - gone # noqa: PLR6104 + + if alive: + # Last attempt over processes survived so far. + # timeout == 0 won't make this function wait any further. + for proc in alive: + check_gone(proc, 0) + alive = alive - gone # noqa: PLR6104 + + return (list(gone), list(alive)) + + +# ===================================================================== +# --- CPU related functions +# ===================================================================== + + +def cpu_count(logical=True): + """Return the number of logical CPUs in the system (same as + os.cpu_count()). + + If *logical* is False return the number of physical cores only + (e.g. hyper thread CPUs are excluded). + + Return None if undetermined. + + The return value is cached after first call. + If desired cache can be cleared like this: + + >>> psutil.cpu_count.cache_clear() + """ + if logical: + ret = _psplatform.cpu_count_logical() + else: + ret = _psplatform.cpu_count_cores() + if ret is not None and ret < 1: + ret = None + return ret + + +def cpu_times(percpu=False): + """Return system-wide CPU times as a namedtuple. + Every CPU time represents the seconds the CPU has spent in the + given mode. The namedtuple's fields availability varies depending on the + platform: + + - user + - system + - idle + - nice (UNIX) + - iowait (Linux) + - irq (Linux, FreeBSD) + - softirq (Linux) + - steal (Linux >= 2.6.11) + - guest (Linux >= 2.6.24) + - guest_nice (Linux >= 3.2.0) + + When *percpu* is True return a list of namedtuples for each CPU. + First element of the list refers to first CPU, second element + to second CPU and so on. + The order of the list is consistent across calls. + """ + if not percpu: + return _psplatform.cpu_times() + else: + return _psplatform.per_cpu_times() + + +try: + _last_cpu_times = {threading.current_thread().ident: cpu_times()} +except Exception: # noqa: BLE001 + # Don't want to crash at import time. + _last_cpu_times = {} + +try: + _last_per_cpu_times = { + threading.current_thread().ident: cpu_times(percpu=True) + } +except Exception: # noqa: BLE001 + # Don't want to crash at import time. + _last_per_cpu_times = {} + + +def _cpu_tot_time(times): + """Given a cpu_time() ntuple calculates the total CPU time + (including idle time). + """ + tot = sum(times) + if LINUX: + # On Linux guest times are already accounted in "user" or + # "nice" times, so we subtract them from total. + # Htop does the same. References: + # https://github.com/giampaolo/psutil/pull/940 + # http://unix.stackexchange.com/questions/178045 + # https://github.com/torvalds/linux/blob/ + # 447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/ + # cputime.c#L158 + tot -= getattr(times, "guest", 0) # Linux 2.6.24+ + tot -= getattr(times, "guest_nice", 0) # Linux 3.2.0+ + return tot + + +def _cpu_busy_time(times): + """Given a cpu_time() ntuple calculates the busy CPU time. + We do so by subtracting all idle CPU times. + """ + busy = _cpu_tot_time(times) + busy -= times.idle + # Linux: "iowait" is time during which the CPU does not do anything + # (waits for IO to complete). On Linux IO wait is *not* accounted + # in "idle" time so we subtract it. Htop does the same. + # References: + # https://github.com/torvalds/linux/blob/ + # 447976ef4fd09b1be88b316d1a81553f1aa7cd07/kernel/sched/cputime.c#L244 + busy -= getattr(times, "iowait", 0) + return busy + + +def _cpu_times_deltas(t1, t2): + assert t1._fields == t2._fields, (t1, t2) + field_deltas = [] + for field in _psplatform.scputimes._fields: + field_delta = getattr(t2, field) - getattr(t1, field) + # CPU times are always supposed to increase over time + # or at least remain the same and that's because time + # cannot go backwards. + # Surprisingly sometimes this might not be the case (at + # least on Windows and Linux), see: + # https://github.com/giampaolo/psutil/issues/392 + # https://github.com/giampaolo/psutil/issues/645 + # https://github.com/giampaolo/psutil/issues/1210 + # Trim negative deltas to zero to ignore decreasing fields. + # top does the same. Reference: + # https://gitlab.com/procps-ng/procps/blob/v3.3.12/top/top.c#L5063 + field_delta = max(0, field_delta) + field_deltas.append(field_delta) + return _psplatform.scputimes(*field_deltas) + + +def cpu_percent(interval=None, percpu=False): + """Return a float representing the current system-wide CPU + utilization as a percentage. + + When *interval* is > 0.0 compares system CPU times elapsed before + and after the interval (blocking). + + When *interval* is 0.0 or None compares system CPU times elapsed + since last call or module import, returning immediately (non + blocking). That means the first time this is called it will + return a meaningless 0.0 value which you should ignore. + In this case is recommended for accuracy that this function be + called with at least 0.1 seconds between calls. + + When *percpu* is True returns a list of floats representing the + utilization as a percentage for each CPU. + First element of the list refers to first CPU, second element + to second CPU and so on. + The order of the list is consistent across calls. + + Examples: + + >>> # blocking, system-wide + >>> psutil.cpu_percent(interval=1) + 2.0 + >>> + >>> # blocking, per-cpu + >>> psutil.cpu_percent(interval=1, percpu=True) + [2.0, 1.0] + >>> + >>> # non-blocking (percentage since last call) + >>> psutil.cpu_percent(interval=None) + 2.9 + >>> + """ + tid = threading.current_thread().ident + blocking = interval is not None and interval > 0.0 + if interval is not None and interval < 0: + msg = f"interval is not positive (got {interval})" + raise ValueError(msg) + + def calculate(t1, t2): + times_delta = _cpu_times_deltas(t1, t2) + all_delta = _cpu_tot_time(times_delta) + busy_delta = _cpu_busy_time(times_delta) + + try: + busy_perc = (busy_delta / all_delta) * 100 + except ZeroDivisionError: + return 0.0 + else: + return round(busy_perc, 1) + + # system-wide usage + if not percpu: + if blocking: + t1 = cpu_times() + time.sleep(interval) + else: + t1 = _last_cpu_times.get(tid) or cpu_times() + _last_cpu_times[tid] = cpu_times() + return calculate(t1, _last_cpu_times[tid]) + # per-cpu usage + else: + ret = [] + if blocking: + tot1 = cpu_times(percpu=True) + time.sleep(interval) + else: + tot1 = _last_per_cpu_times.get(tid) or cpu_times(percpu=True) + _last_per_cpu_times[tid] = cpu_times(percpu=True) + for t1, t2 in zip(tot1, _last_per_cpu_times[tid]): + ret.append(calculate(t1, t2)) + return ret + + +# Use a separate dict for cpu_times_percent(), so it's independent from +# cpu_percent() and they can both be used within the same program. +_last_cpu_times_2 = _last_cpu_times.copy() +_last_per_cpu_times_2 = _last_per_cpu_times.copy() + + +def cpu_times_percent(interval=None, percpu=False): + """Same as cpu_percent() but provides utilization percentages + for each specific CPU time as is returned by cpu_times(). + For instance, on Linux we'll get: + + >>> cpu_times_percent() + cpupercent(user=4.8, nice=0.0, system=4.8, idle=90.5, iowait=0.0, + irq=0.0, softirq=0.0, steal=0.0, guest=0.0, guest_nice=0.0) + >>> + + *interval* and *percpu* arguments have the same meaning as in + cpu_percent(). + """ + tid = threading.current_thread().ident + blocking = interval is not None and interval > 0.0 + if interval is not None and interval < 0: + msg = f"interval is not positive (got {interval!r})" + raise ValueError(msg) + + def calculate(t1, t2): + nums = [] + times_delta = _cpu_times_deltas(t1, t2) + all_delta = _cpu_tot_time(times_delta) + # "scale" is the value to multiply each delta with to get percentages. + # We use "max" to avoid division by zero (if all_delta is 0, then all + # fields are 0 so percentages will be 0 too. all_delta cannot be a + # fraction because cpu times are integers) + scale = 100.0 / max(1, all_delta) + for field_delta in times_delta: + field_perc = field_delta * scale + field_perc = round(field_perc, 1) + # make sure we don't return negative values or values over 100% + field_perc = min(max(0.0, field_perc), 100.0) + nums.append(field_perc) + return _psplatform.scputimes(*nums) + + # system-wide usage + if not percpu: + if blocking: + t1 = cpu_times() + time.sleep(interval) + else: + t1 = _last_cpu_times_2.get(tid) or cpu_times() + _last_cpu_times_2[tid] = cpu_times() + return calculate(t1, _last_cpu_times_2[tid]) + # per-cpu usage + else: + ret = [] + if blocking: + tot1 = cpu_times(percpu=True) + time.sleep(interval) + else: + tot1 = _last_per_cpu_times_2.get(tid) or cpu_times(percpu=True) + _last_per_cpu_times_2[tid] = cpu_times(percpu=True) + for t1, t2 in zip(tot1, _last_per_cpu_times_2[tid]): + ret.append(calculate(t1, t2)) + return ret + + +def cpu_stats(): + """Return CPU statistics.""" + return _psplatform.cpu_stats() + + +if hasattr(_psplatform, "cpu_freq"): + + def cpu_freq(percpu=False): + """Return CPU frequency as a namedtuple including current, + min and max frequency expressed in Mhz. + + If *percpu* is True and the system supports per-cpu frequency + retrieval (Linux only) a list of frequencies is returned for + each CPU. If not a list with one element is returned. + """ + ret = _psplatform.cpu_freq() + if percpu: + return ret + else: + num_cpus = float(len(ret)) + if num_cpus == 0: + return None + elif num_cpus == 1: + return ret[0] + else: + currs, mins, maxs = 0.0, 0.0, 0.0 + set_none = False + for cpu in ret: + currs += cpu.current + # On Linux if /proc/cpuinfo is used min/max are set + # to None. + if LINUX and cpu.min is None: + set_none = True + continue + mins += cpu.min + maxs += cpu.max + + current = currs / num_cpus + + if set_none: + min_ = max_ = None + else: + min_ = mins / num_cpus + max_ = maxs / num_cpus + + return _common.scpufreq(current, min_, max_) + + __all__.append("cpu_freq") + + +if hasattr(os, "getloadavg") or hasattr(_psplatform, "getloadavg"): + # Perform this hasattr check once on import time to either use the + # platform based code or proxy straight from the os module. + if hasattr(os, "getloadavg"): + getloadavg = os.getloadavg + else: + getloadavg = _psplatform.getloadavg + + __all__.append("getloadavg") + + +# ===================================================================== +# --- system memory related functions +# ===================================================================== + + +def virtual_memory(): + """Return statistics about system memory usage as a namedtuple + including the following fields, expressed in bytes: + + - total: + total physical memory available. + + - available: + the memory that can be given instantly to processes without the + system going into swap. + This is calculated by summing different memory values depending + on the platform and it is supposed to be used to monitor actual + memory usage in a cross platform fashion. + + - percent: + the percentage usage calculated as (total - available) / total * 100 + + - used: + memory used, calculated differently depending on the platform and + designed for informational purposes only: + macOS: active + wired + BSD: active + wired + cached + Linux: total - free + + - free: + memory not being used at all (zeroed) that is readily available; + note that this doesn't reflect the actual memory available + (use 'available' instead) + + Platform-specific fields: + + - active (UNIX): + memory currently in use or very recently used, and so it is in RAM. + + - inactive (UNIX): + memory that is marked as not used. + + - buffers (BSD, Linux): + cache for things like file system metadata. + + - cached (BSD, macOS): + cache for various things. + + - wired (macOS, BSD): + memory that is marked to always stay in RAM. It is never moved to disk. + + - shared (BSD): + memory that may be simultaneously accessed by multiple processes. + + The sum of 'used' and 'available' does not necessarily equal total. + On Windows 'available' and 'free' are the same. + """ + global _TOTAL_PHYMEM + ret = _psplatform.virtual_memory() + # cached for later use in Process.memory_percent() + _TOTAL_PHYMEM = ret.total + return ret + + +def swap_memory(): + """Return system swap memory statistics as a namedtuple including + the following fields: + + - total: total swap memory in bytes + - used: used swap memory in bytes + - free: free swap memory in bytes + - percent: the percentage usage + - sin: no. of bytes the system has swapped in from disk (cumulative) + - sout: no. of bytes the system has swapped out from disk (cumulative) + + 'sin' and 'sout' on Windows are meaningless and always set to 0. + """ + return _psplatform.swap_memory() + + +# ===================================================================== +# --- disks/partitions related functions +# ===================================================================== + + +def disk_usage(path): + """Return disk usage statistics about the given *path* as a + namedtuple including total, used and free space expressed in bytes + plus the percentage usage. + """ + return _psplatform.disk_usage(path) + + +def disk_partitions(all=False): + """Return mounted partitions as a list of + (device, mountpoint, fstype, opts) namedtuple. + 'opts' field is a raw string separated by commas indicating mount + options which may vary depending on the platform. + + If *all* parameter is False return physical devices only and ignore + all others. + """ + return _psplatform.disk_partitions(all) + + +def disk_io_counters(perdisk=False, nowrap=True): + """Return system disk I/O statistics as a namedtuple including + the following fields: + + - read_count: number of reads + - write_count: number of writes + - read_bytes: number of bytes read + - write_bytes: number of bytes written + - read_time: time spent reading from disk (in ms) + - write_time: time spent writing to disk (in ms) + + Platform specific: + + - busy_time: (Linux, FreeBSD) time spent doing actual I/Os (in ms) + - read_merged_count (Linux): number of merged reads + - write_merged_count (Linux): number of merged writes + + If *perdisk* is True return the same information for every + physical disk installed on the system as a dictionary + with partition names as the keys and the namedtuple + described above as the values. + + If *nowrap* is True it detects and adjust the numbers which overflow + and wrap (restart from 0) and add "old value" to "new value" so that + the returned numbers will always be increasing or remain the same, + but never decrease. + "disk_io_counters.cache_clear()" can be used to invalidate the + cache. + + On recent Windows versions 'diskperf -y' command may need to be + executed first otherwise this function won't find any disk. + """ + kwargs = dict(perdisk=perdisk) if LINUX else {} + rawdict = _psplatform.disk_io_counters(**kwargs) + if not rawdict: + return {} if perdisk else None + if nowrap: + rawdict = _wrap_numbers(rawdict, 'psutil.disk_io_counters') + nt = getattr(_psplatform, "sdiskio", _common.sdiskio) + if perdisk: + for disk, fields in rawdict.items(): + rawdict[disk] = nt(*fields) + return rawdict + else: + return nt(*(sum(x) for x in zip(*rawdict.values()))) + + +disk_io_counters.cache_clear = functools.partial( + _wrap_numbers.cache_clear, 'psutil.disk_io_counters' +) +disk_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache" + + +# ===================================================================== +# --- network related functions +# ===================================================================== + + +def net_io_counters(pernic=False, nowrap=True): + """Return network I/O statistics as a namedtuple including + the following fields: + + - bytes_sent: number of bytes sent + - bytes_recv: number of bytes received + - packets_sent: number of packets sent + - packets_recv: number of packets received + - errin: total number of errors while receiving + - errout: total number of errors while sending + - dropin: total number of incoming packets which were dropped + - dropout: total number of outgoing packets which were dropped + (always 0 on macOS and BSD) + + If *pernic* is True return the same information for every + network interface installed on the system as a dictionary + with network interface names as the keys and the namedtuple + described above as the values. + + If *nowrap* is True it detects and adjust the numbers which overflow + and wrap (restart from 0) and add "old value" to "new value" so that + the returned numbers will always be increasing or remain the same, + but never decrease. + "net_io_counters.cache_clear()" can be used to invalidate the + cache. + """ + rawdict = _psplatform.net_io_counters() + if not rawdict: + return {} if pernic else None + if nowrap: + rawdict = _wrap_numbers(rawdict, 'psutil.net_io_counters') + if pernic: + for nic, fields in rawdict.items(): + rawdict[nic] = _common.snetio(*fields) + return rawdict + else: + return _common.snetio(*[sum(x) for x in zip(*rawdict.values())]) + + +net_io_counters.cache_clear = functools.partial( + _wrap_numbers.cache_clear, 'psutil.net_io_counters' +) +net_io_counters.cache_clear.__doc__ = "Clears nowrap argument cache" + + +def net_connections(kind='inet'): + """Return system-wide socket connections as a list of + (fd, family, type, laddr, raddr, status, pid) namedtuples. + In case of limited privileges 'fd' and 'pid' may be set to -1 + and None respectively. + The *kind* parameter filters for connections that fit the + following criteria: + + +------------+----------------------------------------------------+ + | Kind Value | Connections using | + +------------+----------------------------------------------------+ + | inet | IPv4 and IPv6 | + | inet4 | IPv4 | + | inet6 | IPv6 | + | tcp | TCP | + | tcp4 | TCP over IPv4 | + | tcp6 | TCP over IPv6 | + | udp | UDP | + | udp4 | UDP over IPv4 | + | udp6 | UDP over IPv6 | + | unix | UNIX socket (both UDP and TCP protocols) | + | all | the sum of all the possible families and protocols | + +------------+----------------------------------------------------+ + + On macOS this function requires root privileges. + """ + _check_conn_kind(kind) + return _psplatform.net_connections(kind) + + +def net_if_addrs(): + """Return the addresses associated to each NIC (network interface + card) installed on the system as a dictionary whose keys are the + NIC names and value is a list of namedtuples for each address + assigned to the NIC. Each namedtuple includes 5 fields: + + - family: can be either socket.AF_INET, socket.AF_INET6 or + psutil.AF_LINK, which refers to a MAC address. + - address: is the primary address and it is always set. + - netmask: and 'broadcast' and 'ptp' may be None. + - ptp: stands for "point to point" and references the + destination address on a point to point interface + (typically a VPN). + - broadcast: and *ptp* are mutually exclusive. + + Note: you can have more than one address of the same family + associated with each interface. + """ + rawlist = _psplatform.net_if_addrs() + rawlist.sort(key=lambda x: x[1]) # sort by family + ret = collections.defaultdict(list) + for name, fam, addr, mask, broadcast, ptp in rawlist: + try: + fam = socket.AddressFamily(fam) + except ValueError: + if WINDOWS and fam == -1: + fam = _psplatform.AF_LINK + elif ( + hasattr(_psplatform, "AF_LINK") and fam == _psplatform.AF_LINK + ): + # Linux defines AF_LINK as an alias for AF_PACKET. + # We re-set the family here so that repr(family) + # will show AF_LINK rather than AF_PACKET + fam = _psplatform.AF_LINK + + if fam == _psplatform.AF_LINK: + # The underlying C function may return an incomplete MAC + # address in which case we fill it with null bytes, see: + # https://github.com/giampaolo/psutil/issues/786 + separator = ":" if POSIX else "-" + while addr.count(separator) < 5: + addr += f"{separator}00" + + nt = _common.snicaddr(fam, addr, mask, broadcast, ptp) + + # On Windows broadcast is None, so we determine it via + # ipaddress module. + if WINDOWS and fam in {socket.AF_INET, socket.AF_INET6}: + try: + broadcast = _common.broadcast_addr(nt) + except Exception as err: # noqa: BLE001 + debug(err) + else: + if broadcast is not None: + nt._replace(broadcast=broadcast) + + ret[name].append(nt) + + return dict(ret) + + +def net_if_stats(): + """Return information about each NIC (network interface card) + installed on the system as a dictionary whose keys are the + NIC names and value is a namedtuple with the following fields: + + - isup: whether the interface is up (bool) + - duplex: can be either NIC_DUPLEX_FULL, NIC_DUPLEX_HALF or + NIC_DUPLEX_UNKNOWN + - speed: the NIC speed expressed in mega bits (MB); if it can't + be determined (e.g. 'localhost') it will be set to 0. + - mtu: the maximum transmission unit expressed in bytes. + """ + return _psplatform.net_if_stats() + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +# Linux, macOS +if hasattr(_psplatform, "sensors_temperatures"): + + def sensors_temperatures(fahrenheit=False): + """Return hardware temperatures. Each entry is a namedtuple + representing a certain hardware sensor (it may be a CPU, an + hard disk or something else, depending on the OS and its + configuration). + All temperatures are expressed in celsius unless *fahrenheit* + is set to True. + """ + + def convert(n): + if n is not None: + return (float(n) * 9 / 5) + 32 if fahrenheit else n + + ret = collections.defaultdict(list) + rawdict = _psplatform.sensors_temperatures() + + for name, values in rawdict.items(): + while values: + label, current, high, critical = values.pop(0) + current = convert(current) + high = convert(high) + critical = convert(critical) + + if high and not critical: + critical = high + elif critical and not high: + high = critical + + ret[name].append( + _common.shwtemp(label, current, high, critical) + ) + + return dict(ret) + + __all__.append("sensors_temperatures") + + +# Linux +if hasattr(_psplatform, "sensors_fans"): + + def sensors_fans(): + """Return fans speed. Each entry is a namedtuple + representing a certain hardware sensor. + All speed are expressed in RPM (rounds per minute). + """ + return _psplatform.sensors_fans() + + __all__.append("sensors_fans") + + +# Linux, Windows, FreeBSD, macOS +if hasattr(_psplatform, "sensors_battery"): + + def sensors_battery(): + """Return battery information. If no battery is installed + returns None. + + - percent: battery power left as a percentage. + - secsleft: a rough approximation of how many seconds are left + before the battery runs out of power. May be + POWER_TIME_UNLIMITED or POWER_TIME_UNLIMITED. + - power_plugged: True if the AC power cable is connected. + """ + return _psplatform.sensors_battery() + + __all__.append("sensors_battery") + + +# ===================================================================== +# --- other system related functions +# ===================================================================== + + +def boot_time(): + """Return the system boot time expressed in seconds since the epoch.""" + # Note: we are not caching this because it is subject to + # system clock updates. + return _psplatform.boot_time() + + +def users(): + """Return users currently connected on the system as a list of + namedtuples including the following fields. + + - user: the name of the user + - terminal: the tty or pseudo-tty associated with the user, if any. + - host: the host name associated with the entry, if any. + - started: the creation time as a floating point number expressed in + seconds since the epoch. + """ + return _psplatform.users() + + +# ===================================================================== +# --- Windows services +# ===================================================================== + + +if WINDOWS: + + def win_service_iter(): + """Return a generator yielding a WindowsService instance for all + Windows services installed. + """ + return _psplatform.win_service_iter() + + def win_service_get(name): + """Get a Windows service by *name*. + Raise NoSuchProcess if no service with such name exists. + """ + return _psplatform.win_service_get(name) + + +# ===================================================================== + + +def _set_debug(value): + """Enable or disable PSUTIL_DEBUG option, which prints debugging + messages to stderr. + """ + import psutil._common + + psutil._common.PSUTIL_DEBUG = bool(value) + _psplatform.cext.set_debug(bool(value)) + + +del memoize_when_activated diff --git a/lib/python3.12/site-packages/psutil/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..18ba0d5027fd3c9ef1c1a45be5ad30d31dc13a7b Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_common.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..04d837656a53c1b8e32ce56224ae8cd1cb1c41c5 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_common.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_psaix.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_psaix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..558ab5eb7c89be0d53bdef05ab2d5b240b2c54cb Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_psaix.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_psbsd.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_psbsd.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..419f1597fe679c53ec3dc86f23ff80875093a354 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_psbsd.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_pslinux.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_pslinux.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3503e7019464643ddf469d0d885b0031450bb1af Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_pslinux.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_psosx.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_psosx.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..722f03179a21d1f320f56b8bc413aa82b3f6659c Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_psosx.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_psposix.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_psposix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6a21aa67c65b6cfa8498b5ee347b9c8d8e6d586 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_psposix.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_pssunos.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_pssunos.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63c5f05dcb01db1d7235f33506b86ea1119c40aa Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_pssunos.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/__pycache__/_pswindows.cpython-312.pyc b/lib/python3.12/site-packages/psutil/__pycache__/_pswindows.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b063527d67493a670113c1152837fe105438dc36 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/__pycache__/_pswindows.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/_common.py b/lib/python3.12/site-packages/psutil/_common.py new file mode 100644 index 0000000000000000000000000000000000000000..4096c0a18c8f8d665cba842d6898bac51bd48f0b --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_common.py @@ -0,0 +1,950 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Common objects shared by __init__.py and _ps*.py modules. + +Note: this module is imported by setup.py, so it should not import +psutil or third-party modules. +""" + +import collections +import enum +import functools +import os +import socket +import stat +import sys +import threading +import warnings +from collections import namedtuple +from socket import AF_INET +from socket import SOCK_DGRAM +from socket import SOCK_STREAM + + +try: + from socket import AF_INET6 +except ImportError: + AF_INET6 = None +try: + from socket import AF_UNIX +except ImportError: + AF_UNIX = None + + +PSUTIL_DEBUG = bool(os.getenv('PSUTIL_DEBUG')) +_DEFAULT = object() + +# fmt: off +__all__ = [ + # OS constants + 'FREEBSD', 'BSD', 'LINUX', 'NETBSD', 'OPENBSD', 'MACOS', 'OSX', 'POSIX', + 'SUNOS', 'WINDOWS', + # connection constants + 'CONN_CLOSE', 'CONN_CLOSE_WAIT', 'CONN_CLOSING', 'CONN_ESTABLISHED', + 'CONN_FIN_WAIT1', 'CONN_FIN_WAIT2', 'CONN_LAST_ACK', 'CONN_LISTEN', + 'CONN_NONE', 'CONN_SYN_RECV', 'CONN_SYN_SENT', 'CONN_TIME_WAIT', + # net constants + 'NIC_DUPLEX_FULL', 'NIC_DUPLEX_HALF', 'NIC_DUPLEX_UNKNOWN', # noqa: F822 + # process status constants + 'STATUS_DEAD', 'STATUS_DISK_SLEEP', 'STATUS_IDLE', 'STATUS_LOCKED', + 'STATUS_RUNNING', 'STATUS_SLEEPING', 'STATUS_STOPPED', 'STATUS_SUSPENDED', + 'STATUS_TRACING_STOP', 'STATUS_WAITING', 'STATUS_WAKE_KILL', + 'STATUS_WAKING', 'STATUS_ZOMBIE', 'STATUS_PARKED', + # other constants + 'ENCODING', 'ENCODING_ERRS', 'AF_INET6', + # named tuples + 'pconn', 'pcputimes', 'pctxsw', 'pgids', 'pio', 'pionice', 'popenfile', + 'pthread', 'puids', 'sconn', 'scpustats', 'sdiskio', 'sdiskpart', + 'sdiskusage', 'snetio', 'snicaddr', 'snicstats', 'sswap', 'suser', + # utility functions + 'conn_tmap', 'deprecated_method', 'isfile_strict', 'memoize', + 'parse_environ_block', 'path_exists_strict', 'usage_percent', + 'supports_ipv6', 'sockfam_to_enum', 'socktype_to_enum', "wrap_numbers", + 'open_text', 'open_binary', 'cat', 'bcat', + 'bytes2human', 'conn_to_ntuple', 'debug', + # shell utils + 'hilite', 'term_supports_colors', 'print_color', +] +# fmt: on + + +# =================================================================== +# --- OS constants +# =================================================================== + + +POSIX = os.name == "posix" +WINDOWS = os.name == "nt" +LINUX = sys.platform.startswith("linux") +MACOS = sys.platform.startswith("darwin") +OSX = MACOS # deprecated alias +FREEBSD = sys.platform.startswith(("freebsd", "midnightbsd")) +OPENBSD = sys.platform.startswith("openbsd") +NETBSD = sys.platform.startswith("netbsd") +BSD = FREEBSD or OPENBSD or NETBSD +SUNOS = sys.platform.startswith(("sunos", "solaris")) +AIX = sys.platform.startswith("aix") + + +# =================================================================== +# --- API constants +# =================================================================== + + +# Process.status() +STATUS_RUNNING = "running" +STATUS_SLEEPING = "sleeping" +STATUS_DISK_SLEEP = "disk-sleep" +STATUS_STOPPED = "stopped" +STATUS_TRACING_STOP = "tracing-stop" +STATUS_ZOMBIE = "zombie" +STATUS_DEAD = "dead" +STATUS_WAKE_KILL = "wake-kill" +STATUS_WAKING = "waking" +STATUS_IDLE = "idle" # Linux, macOS, FreeBSD +STATUS_LOCKED = "locked" # FreeBSD +STATUS_WAITING = "waiting" # FreeBSD +STATUS_SUSPENDED = "suspended" # NetBSD +STATUS_PARKED = "parked" # Linux + +# Process.net_connections() and psutil.net_connections() +CONN_ESTABLISHED = "ESTABLISHED" +CONN_SYN_SENT = "SYN_SENT" +CONN_SYN_RECV = "SYN_RECV" +CONN_FIN_WAIT1 = "FIN_WAIT1" +CONN_FIN_WAIT2 = "FIN_WAIT2" +CONN_TIME_WAIT = "TIME_WAIT" +CONN_CLOSE = "CLOSE" +CONN_CLOSE_WAIT = "CLOSE_WAIT" +CONN_LAST_ACK = "LAST_ACK" +CONN_LISTEN = "LISTEN" +CONN_CLOSING = "CLOSING" +CONN_NONE = "NONE" + + +# net_if_stats() +class NicDuplex(enum.IntEnum): + NIC_DUPLEX_FULL = 2 + NIC_DUPLEX_HALF = 1 + NIC_DUPLEX_UNKNOWN = 0 + + +globals().update(NicDuplex.__members__) + + +# sensors_battery() +class BatteryTime(enum.IntEnum): + POWER_TIME_UNKNOWN = -1 + POWER_TIME_UNLIMITED = -2 + + +globals().update(BatteryTime.__members__) + +# --- others + +ENCODING = sys.getfilesystemencoding() +ENCODING_ERRS = sys.getfilesystemencodeerrors() + + +# =================================================================== +# --- namedtuples +# =================================================================== + +# --- for system functions + +# fmt: off +# psutil.swap_memory() +sswap = namedtuple('sswap', ['total', 'used', 'free', 'percent', 'sin', + 'sout']) +# psutil.disk_usage() +sdiskusage = namedtuple('sdiskusage', ['total', 'used', 'free', 'percent']) +# psutil.disk_io_counters() +sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_time', 'write_time']) +# psutil.disk_partitions() +sdiskpart = namedtuple('sdiskpart', ['device', 'mountpoint', 'fstype', 'opts']) +# psutil.net_io_counters() +snetio = namedtuple('snetio', ['bytes_sent', 'bytes_recv', + 'packets_sent', 'packets_recv', + 'errin', 'errout', + 'dropin', 'dropout']) +# psutil.users() +suser = namedtuple('suser', ['name', 'terminal', 'host', 'started', 'pid']) +# psutil.net_connections() +sconn = namedtuple('sconn', ['fd', 'family', 'type', 'laddr', 'raddr', + 'status', 'pid']) +# psutil.net_if_addrs() +snicaddr = namedtuple('snicaddr', + ['family', 'address', 'netmask', 'broadcast', 'ptp']) +# psutil.net_if_stats() +snicstats = namedtuple('snicstats', + ['isup', 'duplex', 'speed', 'mtu', 'flags']) +# psutil.cpu_stats() +scpustats = namedtuple( + 'scpustats', ['ctx_switches', 'interrupts', 'soft_interrupts', 'syscalls']) +# psutil.cpu_freq() +scpufreq = namedtuple('scpufreq', ['current', 'min', 'max']) +# psutil.sensors_temperatures() +shwtemp = namedtuple( + 'shwtemp', ['label', 'current', 'high', 'critical']) +# psutil.sensors_battery() +sbattery = namedtuple('sbattery', ['percent', 'secsleft', 'power_plugged']) +# psutil.sensors_fans() +sfan = namedtuple('sfan', ['label', 'current']) +# fmt: on + +# --- for Process methods + +# psutil.Process.cpu_times() +pcputimes = namedtuple( + 'pcputimes', ['user', 'system', 'children_user', 'children_system'] +) +# psutil.Process.open_files() +popenfile = namedtuple('popenfile', ['path', 'fd']) +# psutil.Process.threads() +pthread = namedtuple('pthread', ['id', 'user_time', 'system_time']) +# psutil.Process.uids() +puids = namedtuple('puids', ['real', 'effective', 'saved']) +# psutil.Process.gids() +pgids = namedtuple('pgids', ['real', 'effective', 'saved']) +# psutil.Process.io_counters() +pio = namedtuple( + 'pio', ['read_count', 'write_count', 'read_bytes', 'write_bytes'] +) +# psutil.Process.ionice() +pionice = namedtuple('pionice', ['ioclass', 'value']) +# psutil.Process.ctx_switches() +pctxsw = namedtuple('pctxsw', ['voluntary', 'involuntary']) +# psutil.Process.net_connections() +pconn = namedtuple( + 'pconn', ['fd', 'family', 'type', 'laddr', 'raddr', 'status'] +) + +# psutil.net_connections() and psutil.Process.net_connections() +addr = namedtuple('addr', ['ip', 'port']) + + +# =================================================================== +# --- Process.net_connections() 'kind' parameter mapping +# =================================================================== + + +conn_tmap = { + "all": ([AF_INET, AF_INET6, AF_UNIX], [SOCK_STREAM, SOCK_DGRAM]), + "tcp": ([AF_INET, AF_INET6], [SOCK_STREAM]), + "tcp4": ([AF_INET], [SOCK_STREAM]), + "udp": ([AF_INET, AF_INET6], [SOCK_DGRAM]), + "udp4": ([AF_INET], [SOCK_DGRAM]), + "inet": ([AF_INET, AF_INET6], [SOCK_STREAM, SOCK_DGRAM]), + "inet4": ([AF_INET], [SOCK_STREAM, SOCK_DGRAM]), + "inet6": ([AF_INET6], [SOCK_STREAM, SOCK_DGRAM]), +} + +if AF_INET6 is not None: + conn_tmap.update({ + "tcp6": ([AF_INET6], [SOCK_STREAM]), + "udp6": ([AF_INET6], [SOCK_DGRAM]), + }) + +if AF_UNIX is not None and not SUNOS: + conn_tmap.update({"unix": ([AF_UNIX], [SOCK_STREAM, SOCK_DGRAM])}) + + +# ===================================================================== +# --- Exceptions +# ===================================================================== + + +class Error(Exception): + """Base exception class. All other psutil exceptions inherit + from this one. + """ + + __module__ = 'psutil' + + def _infodict(self, attrs): + info = collections.OrderedDict() + for name in attrs: + value = getattr(self, name, None) + if value or (name == "pid" and value == 0): + info[name] = value + return info + + def __str__(self): + # invoked on `raise Error` + info = self._infodict(("pid", "ppid", "name")) + if info: + details = "({})".format( + ", ".join([f"{k}={v!r}" for k, v in info.items()]) + ) + else: + details = None + return " ".join([x for x in (getattr(self, "msg", ""), details) if x]) + + def __repr__(self): + # invoked on `repr(Error)` + info = self._infodict(("pid", "ppid", "name", "seconds", "msg")) + details = ", ".join([f"{k}={v!r}" for k, v in info.items()]) + return f"psutil.{self.__class__.__name__}({details})" + + +class NoSuchProcess(Error): + """Exception raised when a process with a certain PID doesn't + or no longer exists. + """ + + __module__ = 'psutil' + + def __init__(self, pid, name=None, msg=None): + Error.__init__(self) + self.pid = pid + self.name = name + self.msg = msg or "process no longer exists" + + def __reduce__(self): + return (self.__class__, (self.pid, self.name, self.msg)) + + +class ZombieProcess(NoSuchProcess): + """Exception raised when querying a zombie process. This is + raised on macOS, BSD and Solaris only, and not always: depending + on the query the OS may be able to succeed anyway. + On Linux all zombie processes are querable (hence this is never + raised). Windows doesn't have zombie processes. + """ + + __module__ = 'psutil' + + def __init__(self, pid, name=None, ppid=None, msg=None): + NoSuchProcess.__init__(self, pid, name, msg) + self.ppid = ppid + self.msg = msg or "PID still exists but it's a zombie" + + def __reduce__(self): + return (self.__class__, (self.pid, self.name, self.ppid, self.msg)) + + +class AccessDenied(Error): + """Exception raised when permission to perform an action is denied.""" + + __module__ = 'psutil' + + def __init__(self, pid=None, name=None, msg=None): + Error.__init__(self) + self.pid = pid + self.name = name + self.msg = msg or "" + + def __reduce__(self): + return (self.__class__, (self.pid, self.name, self.msg)) + + +class TimeoutExpired(Error): + """Raised on Process.wait(timeout) if timeout expires and process + is still alive. + """ + + __module__ = 'psutil' + + def __init__(self, seconds, pid=None, name=None): + Error.__init__(self) + self.seconds = seconds + self.pid = pid + self.name = name + self.msg = f"timeout after {seconds} seconds" + + def __reduce__(self): + return (self.__class__, (self.seconds, self.pid, self.name)) + + +# =================================================================== +# --- utils +# =================================================================== + + +def usage_percent(used, total, round_=None): + """Calculate percentage usage of 'used' against 'total'.""" + try: + ret = (float(used) / total) * 100 + except ZeroDivisionError: + return 0.0 + else: + if round_ is not None: + ret = round(ret, round_) + return ret + + +def memoize(fun): + """A simple memoize decorator for functions supporting (hashable) + positional arguments. + It also provides a cache_clear() function for clearing the cache: + + >>> @memoize + ... def foo() + ... return 1 + ... + >>> foo() + 1 + >>> foo.cache_clear() + >>> + + It supports: + - functions + - classes (acts as a @singleton) + - staticmethods + - classmethods + + It does NOT support: + - methods + """ + + @functools.wraps(fun) + def wrapper(*args, **kwargs): + key = (args, frozenset(sorted(kwargs.items()))) + try: + return cache[key] + except KeyError: + try: + ret = cache[key] = fun(*args, **kwargs) + except Exception as err: # noqa: BLE001 + raise err from None + return ret + + def cache_clear(): + """Clear cache.""" + cache.clear() + + cache = {} + wrapper.cache_clear = cache_clear + return wrapper + + +def memoize_when_activated(fun): + """A memoize decorator which is disabled by default. It can be + activated and deactivated on request. + For efficiency reasons it can be used only against class methods + accepting no arguments. + + >>> class Foo: + ... @memoize + ... def foo() + ... print(1) + ... + >>> f = Foo() + >>> # deactivated (default) + >>> foo() + 1 + >>> foo() + 1 + >>> + >>> # activated + >>> foo.cache_activate(self) + >>> foo() + 1 + >>> foo() + >>> foo() + >>> + """ + + @functools.wraps(fun) + def wrapper(self): + try: + # case 1: we previously entered oneshot() ctx + ret = self._cache[fun] + except AttributeError: + # case 2: we never entered oneshot() ctx + try: + return fun(self) + except Exception as err: # noqa: BLE001 + raise err from None + except KeyError: + # case 3: we entered oneshot() ctx but there's no cache + # for this entry yet + try: + ret = fun(self) + except Exception as err: # noqa: BLE001 + raise err from None + try: + self._cache[fun] = ret + except AttributeError: + # multi-threading race condition, see: + # https://github.com/giampaolo/psutil/issues/1948 + pass + return ret + + def cache_activate(proc): + """Activate cache. Expects a Process instance. Cache will be + stored as a "_cache" instance attribute. + """ + proc._cache = {} + + def cache_deactivate(proc): + """Deactivate and clear cache.""" + try: + del proc._cache + except AttributeError: + pass + + wrapper.cache_activate = cache_activate + wrapper.cache_deactivate = cache_deactivate + return wrapper + + +def isfile_strict(path): + """Same as os.path.isfile() but does not swallow EACCES / EPERM + exceptions, see: + http://mail.python.org/pipermail/python-dev/2012-June/120787.html. + """ + try: + st = os.stat(path) + except PermissionError: + raise + except OSError: + return False + else: + return stat.S_ISREG(st.st_mode) + + +def path_exists_strict(path): + """Same as os.path.exists() but does not swallow EACCES / EPERM + exceptions. See: + http://mail.python.org/pipermail/python-dev/2012-June/120787.html. + """ + try: + os.stat(path) + except PermissionError: + raise + except OSError: + return False + else: + return True + + +@memoize +def supports_ipv6(): + """Return True if IPv6 is supported on this platform.""" + if not socket.has_ipv6 or AF_INET6 is None: + return False + try: + with socket.socket(AF_INET6, socket.SOCK_STREAM) as sock: + sock.bind(("::1", 0)) + return True + except OSError: + return False + + +def parse_environ_block(data): + """Parse a C environ block of environment variables into a dictionary.""" + # The block is usually raw data from the target process. It might contain + # trailing garbage and lines that do not look like assignments. + ret = {} + pos = 0 + + # localize global variable to speed up access. + WINDOWS_ = WINDOWS + while True: + next_pos = data.find("\0", pos) + # nul byte at the beginning or double nul byte means finish + if next_pos <= pos: + break + # there might not be an equals sign + equal_pos = data.find("=", pos, next_pos) + if equal_pos > pos: + key = data[pos:equal_pos] + value = data[equal_pos + 1 : next_pos] + # Windows expects environment variables to be uppercase only + if WINDOWS_: + key = key.upper() + ret[key] = value + pos = next_pos + 1 + + return ret + + +def sockfam_to_enum(num): + """Convert a numeric socket family value to an IntEnum member. + If it's not a known member, return the numeric value itself. + """ + try: + return socket.AddressFamily(num) + except ValueError: + return num + + +def socktype_to_enum(num): + """Convert a numeric socket type value to an IntEnum member. + If it's not a known member, return the numeric value itself. + """ + try: + return socket.SocketKind(num) + except ValueError: + return num + + +def conn_to_ntuple(fd, fam, type_, laddr, raddr, status, status_map, pid=None): + """Convert a raw connection tuple to a proper ntuple.""" + if fam in {socket.AF_INET, AF_INET6}: + if laddr: + laddr = addr(*laddr) + if raddr: + raddr = addr(*raddr) + if type_ == socket.SOCK_STREAM and fam in {AF_INET, AF_INET6}: + status = status_map.get(status, CONN_NONE) + else: + status = CONN_NONE # ignore whatever C returned to us + fam = sockfam_to_enum(fam) + type_ = socktype_to_enum(type_) + if pid is None: + return pconn(fd, fam, type_, laddr, raddr, status) + else: + return sconn(fd, fam, type_, laddr, raddr, status, pid) + + +def broadcast_addr(addr): + """Given the address ntuple returned by ``net_if_addrs()`` + calculates the broadcast address. + """ + import ipaddress + + if not addr.address or not addr.netmask: + return None + if addr.family == socket.AF_INET: + return str( + ipaddress.IPv4Network( + f"{addr.address}/{addr.netmask}", strict=False + ).broadcast_address + ) + if addr.family == socket.AF_INET6: + return str( + ipaddress.IPv6Network( + f"{addr.address}/{addr.netmask}", strict=False + ).broadcast_address + ) + + +def deprecated_method(replacement): + """A decorator which can be used to mark a method as deprecated + 'replcement' is the method name which will be called instead. + """ + + def outer(fun): + msg = ( + f"{fun.__name__}() is deprecated and will be removed; use" + f" {replacement}() instead" + ) + if fun.__doc__ is None: + fun.__doc__ = msg + + @functools.wraps(fun) + def inner(self, *args, **kwargs): + warnings.warn(msg, category=DeprecationWarning, stacklevel=2) + return getattr(self, replacement)(*args, **kwargs) + + return inner + + return outer + + +class _WrapNumbers: + """Watches numbers so that they don't overflow and wrap + (reset to zero). + """ + + def __init__(self): + self.lock = threading.Lock() + self.cache = {} + self.reminders = {} + self.reminder_keys = {} + + def _add_dict(self, input_dict, name): + assert name not in self.cache + assert name not in self.reminders + assert name not in self.reminder_keys + self.cache[name] = input_dict + self.reminders[name] = collections.defaultdict(int) + self.reminder_keys[name] = collections.defaultdict(set) + + def _remove_dead_reminders(self, input_dict, name): + """In case the number of keys changed between calls (e.g. a + disk disappears) this removes the entry from self.reminders. + """ + old_dict = self.cache[name] + gone_keys = set(old_dict.keys()) - set(input_dict.keys()) + for gone_key in gone_keys: + for remkey in self.reminder_keys[name][gone_key]: + del self.reminders[name][remkey] + del self.reminder_keys[name][gone_key] + + def run(self, input_dict, name): + """Cache dict and sum numbers which overflow and wrap. + Return an updated copy of `input_dict`. + """ + if name not in self.cache: + # This was the first call. + self._add_dict(input_dict, name) + return input_dict + + self._remove_dead_reminders(input_dict, name) + + old_dict = self.cache[name] + new_dict = {} + for key in input_dict: + input_tuple = input_dict[key] + try: + old_tuple = old_dict[key] + except KeyError: + # The input dict has a new key (e.g. a new disk or NIC) + # which didn't exist in the previous call. + new_dict[key] = input_tuple + continue + + bits = [] + for i in range(len(input_tuple)): + input_value = input_tuple[i] + old_value = old_tuple[i] + remkey = (key, i) + if input_value < old_value: + # it wrapped! + self.reminders[name][remkey] += old_value + self.reminder_keys[name][key].add(remkey) + bits.append(input_value + self.reminders[name][remkey]) + + new_dict[key] = tuple(bits) + + self.cache[name] = input_dict + return new_dict + + def cache_clear(self, name=None): + """Clear the internal cache, optionally only for function 'name'.""" + with self.lock: + if name is None: + self.cache.clear() + self.reminders.clear() + self.reminder_keys.clear() + else: + self.cache.pop(name, None) + self.reminders.pop(name, None) + self.reminder_keys.pop(name, None) + + def cache_info(self): + """Return internal cache dicts as a tuple of 3 elements.""" + with self.lock: + return (self.cache, self.reminders, self.reminder_keys) + + +def wrap_numbers(input_dict, name): + """Given an `input_dict` and a function `name`, adjust the numbers + which "wrap" (restart from zero) across different calls by adding + "old value" to "new value" and return an updated dict. + """ + with _wn.lock: + return _wn.run(input_dict, name) + + +_wn = _WrapNumbers() +wrap_numbers.cache_clear = _wn.cache_clear +wrap_numbers.cache_info = _wn.cache_info + + +# The read buffer size for open() builtin. This (also) dictates how +# much data we read(2) when iterating over file lines as in: +# >>> with open(file) as f: +# ... for line in f: +# ... ... +# Default per-line buffer size for binary files is 1K. For text files +# is 8K. We use a bigger buffer (32K) in order to have more consistent +# results when reading /proc pseudo files on Linux, see: +# https://github.com/giampaolo/psutil/issues/2050 +# https://github.com/giampaolo/psutil/issues/708 +FILE_READ_BUFFER_SIZE = 32 * 1024 + + +def open_binary(fname): + return open(fname, "rb", buffering=FILE_READ_BUFFER_SIZE) + + +def open_text(fname): + """Open a file in text mode by using the proper FS encoding and + en/decoding error handlers. + """ + # See: + # https://github.com/giampaolo/psutil/issues/675 + # https://github.com/giampaolo/psutil/pull/733 + fobj = open( # noqa: SIM115 + fname, + buffering=FILE_READ_BUFFER_SIZE, + encoding=ENCODING, + errors=ENCODING_ERRS, + ) + try: + # Dictates per-line read(2) buffer size. Defaults is 8k. See: + # https://github.com/giampaolo/psutil/issues/2050#issuecomment-1013387546 + fobj._CHUNK_SIZE = FILE_READ_BUFFER_SIZE + except AttributeError: + pass + except Exception: + fobj.close() + raise + + return fobj + + +def cat(fname, fallback=_DEFAULT, _open=open_text): + """Read entire file content and return it as a string. File is + opened in text mode. If specified, `fallback` is the value + returned in case of error, either if the file does not exist or + it can't be read(). + """ + if fallback is _DEFAULT: + with _open(fname) as f: + return f.read() + else: + try: + with _open(fname) as f: + return f.read() + except OSError: + return fallback + + +def bcat(fname, fallback=_DEFAULT): + """Same as above but opens file in binary mode.""" + return cat(fname, fallback=fallback, _open=open_binary) + + +def bytes2human(n, format="%(value).1f%(symbol)s"): + """Used by various scripts. See: https://code.activestate.com/recipes/578019-bytes-to-human-human-to-bytes-converter/?in=user-4178764. + + >>> bytes2human(10000) + '9.8K' + >>> bytes2human(100001221) + '95.4M' + """ + symbols = ('B', 'K', 'M', 'G', 'T', 'P', 'E', 'Z', 'Y') + prefix = {} + for i, s in enumerate(symbols[1:]): + prefix[s] = 1 << (i + 1) * 10 + for symbol in reversed(symbols[1:]): + if abs(n) >= prefix[symbol]: + value = float(n) / prefix[symbol] + return format % locals() + return format % dict(symbol=symbols[0], value=n) + + +def get_procfs_path(): + """Return updated psutil.PROCFS_PATH constant.""" + return sys.modules['psutil'].PROCFS_PATH + + +def decode(s): + return s.decode(encoding=ENCODING, errors=ENCODING_ERRS) + + +# ===================================================================== +# --- shell utils +# ===================================================================== + + +@memoize +def term_supports_colors(file=sys.stdout): # pragma: no cover + if os.name == 'nt': + return True + try: + import curses + + assert file.isatty() + curses.setupterm() + assert curses.tigetnum("colors") > 0 + except Exception: # noqa: BLE001 + return False + else: + return True + + +def hilite(s, color=None, bold=False): # pragma: no cover + """Return an highlighted version of 'string'.""" + if not term_supports_colors(): + return s + attr = [] + colors = dict( + blue='34', + brown='33', + darkgrey='30', + green='32', + grey='37', + lightblue='36', + red='91', + violet='35', + yellow='93', + ) + colors[None] = '29' + try: + color = colors[color] + except KeyError: + msg = f"invalid color {color!r}; choose amongst {list(colors.keys())}" + raise ValueError(msg) from None + attr.append(color) + if bold: + attr.append('1') + return f"\x1b[{';'.join(attr)}m{s}\x1b[0m" + + +def print_color( + s, color=None, bold=False, file=sys.stdout +): # pragma: no cover + """Print a colorized version of string.""" + if not term_supports_colors(): + print(s, file=file) + elif POSIX: + print(hilite(s, color, bold), file=file) + else: + import ctypes + + DEFAULT_COLOR = 7 + GetStdHandle = ctypes.windll.Kernel32.GetStdHandle + SetConsoleTextAttribute = ( + ctypes.windll.Kernel32.SetConsoleTextAttribute + ) + + colors = dict(green=2, red=4, brown=6, yellow=6) + colors[None] = DEFAULT_COLOR + try: + color = colors[color] + except KeyError: + msg = ( + f"invalid color {color!r}; choose between" + f" {list(colors.keys())!r}" + ) + raise ValueError(msg) from None + if bold and color <= 7: + color += 8 + + handle_id = -12 if file is sys.stderr else -11 + GetStdHandle.restype = ctypes.c_ulong + handle = GetStdHandle(handle_id) + SetConsoleTextAttribute(handle, color) + try: + print(s, file=file) + finally: + SetConsoleTextAttribute(handle, DEFAULT_COLOR) + + +def debug(msg): + """If PSUTIL_DEBUG env var is set, print a debug message to stderr.""" + if PSUTIL_DEBUG: + import inspect + + fname, lineno, _, _lines, _index = inspect.getframeinfo( + inspect.currentframe().f_back + ) + if isinstance(msg, Exception): + if isinstance(msg, OSError): + # ...because str(exc) may contain info about the file name + msg = f"ignoring {msg}" + else: + msg = f"ignoring {msg!r}" + print( # noqa: T201 + f"psutil-debug [{fname}:{lineno}]> {msg}", file=sys.stderr + ) diff --git a/lib/python3.12/site-packages/psutil/_psaix.py b/lib/python3.12/site-packages/psutil/_psaix.py new file mode 100644 index 0000000000000000000000000000000000000000..ba2725fdc10d38532e9784f1efe80fed2210983c --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_psaix.py @@ -0,0 +1,565 @@ +# Copyright (c) 2009, Giampaolo Rodola' +# Copyright (c) 2017, Arnon Yaari +# All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""AIX platform implementation.""" + +import functools +import glob +import os +import re +import subprocess +import sys +from collections import namedtuple + +from . import _common +from . import _psposix +from . import _psutil_aix as cext +from . import _psutil_posix as cext_posix +from ._common import NIC_DUPLEX_FULL +from ._common import NIC_DUPLEX_HALF +from ._common import NIC_DUPLEX_UNKNOWN +from ._common import AccessDenied +from ._common import NoSuchProcess +from ._common import ZombieProcess +from ._common import conn_to_ntuple +from ._common import get_procfs_path +from ._common import memoize_when_activated +from ._common import usage_percent + + +__extra__all__ = ["PROCFS_PATH"] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +HAS_THREADS = hasattr(cext, "proc_threads") +HAS_NET_IO_COUNTERS = hasattr(cext, "net_io_counters") +HAS_PROC_IO_COUNTERS = hasattr(cext, "proc_io_counters") + +PAGE_SIZE = cext_posix.getpagesize() +AF_LINK = cext_posix.AF_LINK + +PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SACTIVE: _common.STATUS_RUNNING, + cext.SSWAP: _common.STATUS_RUNNING, # TODO what status is this? + cext.SSTOP: _common.STATUS_STOPPED, +} + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RCVD: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +proc_info_map = dict( + ppid=0, + rss=1, + vms=2, + create_time=3, + nice=4, + num_threads=5, + status=6, + ttynr=7, +) + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms']) +# psutil.Process.memory_full_info() +pfullmem = pmem +# psutil.Process.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'iowait']) +# psutil.virtual_memory() +svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + total, avail, free, _pinned, inuse = cext.virtual_mem() + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, inuse, free) + + +def swap_memory(): + """Swap system memory as a (total, used, free, sin, sout) tuple.""" + total, free, sin, sout = cext.swap_mem() + used = total - free + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system-wide CPU times as a named tuple.""" + ret = cext.per_cpu_times() + return scputimes(*[sum(x) for x in zip(*ret)]) + + +def per_cpu_times(): + """Return system per-CPU times as a list of named tuples.""" + ret = cext.per_cpu_times() + return [scputimes(*x) for x in ret] + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + try: + return os.sysconf("SC_NPROCESSORS_ONLN") + except ValueError: + # mimic os.cpu_count() behavior + return None + + +def cpu_count_cores(): + cmd = ["lsdev", "-Cc", "processor"] + p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + stdout, stderr = p.communicate() + stdout, stderr = (x.decode(sys.stdout.encoding) for x in (stdout, stderr)) + if p.returncode != 0: + msg = f"{cmd!r} command error\n{stderr}" + raise RuntimeError(msg) + processors = stdout.strip().splitlines() + return len(processors) or None + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + ctx_switches, interrupts, soft_interrupts, syscalls = cext.cpu_stats() + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls + ) + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_io_counters = cext.disk_io_counters +disk_usage = _psposix.disk_usage + + +def disk_partitions(all=False): + """Return system disk partitions.""" + # TODO - the filtering logic should be better checked so that + # it tries to reflect 'df' as much as possible + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + # Differently from, say, Linux, we don't have a list of + # common fs types so the best we can do, AFAIK, is to + # filter by filesystem having a total size > 0. + if not disk_usage(mountpoint).total: + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_if_addrs = cext_posix.net_if_addrs + +if HAS_NET_IO_COUNTERS: + net_io_counters = cext.net_io_counters + + +def net_connections(kind, _pid=-1): + """Return socket connections. If pid == -1 return system-wide + connections (as opposed to connections opened by one process only). + """ + families, types = _common.conn_tmap[kind] + rawlist = cext.net_connections(_pid) + ret = [] + for item in rawlist: + fd, fam, type_, laddr, raddr, status, pid = item + if fam not in families: + continue + if type_ not in types: + continue + nt = conn_to_ntuple( + fd, + fam, + type_, + laddr, + raddr, + status, + TCP_STATUSES, + pid=pid if _pid == -1 else None, + ) + ret.append(nt) + return ret + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + duplex_map = {"Full": NIC_DUPLEX_FULL, "Half": NIC_DUPLEX_HALF} + names = {x[0] for x in net_if_addrs()} + ret = {} + for name in names: + mtu = cext_posix.net_if_mtu(name) + flags = cext_posix.net_if_flags(name) + + # try to get speed and duplex + # TODO: rewrite this in C (entstat forks, so use truss -f to follow. + # looks like it is using an undocumented ioctl?) + duplex = "" + speed = 0 + p = subprocess.Popen( + ["/usr/bin/entstat", "-d", name], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + stdout, stderr = ( + x.decode(sys.stdout.encoding) for x in (stdout, stderr) + ) + if p.returncode == 0: + re_result = re.search( + r"Running: (\d+) Mbps.*?(\w+) Duplex", stdout + ) + if re_result is not None: + speed = int(re_result.group(1)) + duplex = re_result.group(2) + + output_flags = ','.join(flags) + isup = 'running' in flags + duplex = duplex_map.get(duplex, NIC_DUPLEX_UNKNOWN) + ret[name] = _common.snicstats(isup, duplex, speed, mtu, output_flags) + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + localhost = (':0.0', ':0') + for item in rawlist: + user, tty, hostname, tstamp, user_process, pid = item + # note: the underlying C function includes entries about + # system boot, run level and others. We might want + # to use them in the future. + if not user_process: + continue + if hostname in localhost: + hostname = 'localhost' + nt = _common.suser(user, tty, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + return [int(x) for x in os.listdir(get_procfs_path()) if x.isdigit()] + + +def pid_exists(pid): + """Check for the existence of a unix pid.""" + return os.path.exists(os.path.join(get_procfs_path(), str(pid), "psinfo")) + + +def wrap_exceptions(fun): + """Call callable into a try/except clause and translate ENOENT, + EACCES and EPERM in NoSuchProcess or AccessDenied exceptions. + """ + + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name + try: + return fun(self, *args, **kwargs) + except (FileNotFoundError, ProcessLookupError) as err: + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if not pid_exists(pid): + raise NoSuchProcess(pid, name) from err + raise ZombieProcess(pid, name, ppid) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + + return wrapper + + +class Process: + """Wrapper class around underlying C implementation.""" + + __slots__ = ["_cache", "_name", "_ppid", "_procfs_path", "pid"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + self._procfs_path = get_procfs_path() + + def oneshot_enter(self): + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) + + def oneshot_exit(self): + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) + + @wrap_exceptions + @memoize_when_activated + def _proc_basic_info(self): + return cext.proc_basic_info(self.pid, self._procfs_path) + + @wrap_exceptions + @memoize_when_activated + def _proc_cred(self): + return cext.proc_cred(self.pid, self._procfs_path) + + @wrap_exceptions + def name(self): + if self.pid == 0: + return "swapper" + # note: max 16 characters + return cext.proc_name(self.pid, self._procfs_path).rstrip("\x00") + + @wrap_exceptions + def exe(self): + # there is no way to get executable path in AIX other than to guess, + # and guessing is more complex than what's in the wrapping class + cmdline = self.cmdline() + if not cmdline: + return '' + exe = cmdline[0] + if os.path.sep in exe: + # relative or absolute path + if not os.path.isabs(exe): + # if cwd has changed, we're out of luck - this may be wrong! + exe = os.path.abspath(os.path.join(self.cwd(), exe)) + if ( + os.path.isabs(exe) + and os.path.isfile(exe) + and os.access(exe, os.X_OK) + ): + return exe + # not found, move to search in PATH using basename only + exe = os.path.basename(exe) + # search for exe name PATH + for path in os.environ["PATH"].split(":"): + possible_exe = os.path.abspath(os.path.join(path, exe)) + if os.path.isfile(possible_exe) and os.access( + possible_exe, os.X_OK + ): + return possible_exe + return '' + + @wrap_exceptions + def cmdline(self): + return cext.proc_args(self.pid) + + @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid) + + @wrap_exceptions + def create_time(self): + return self._proc_basic_info()[proc_info_map['create_time']] + + @wrap_exceptions + def num_threads(self): + return self._proc_basic_info()[proc_info_map['num_threads']] + + if HAS_THREADS: + + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + # The underlying C implementation retrieves all OS threads + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not retlist: + # will raise NSP if process is gone + os.stat(f"{self._procfs_path}/{self.pid}") + return retlist + + @wrap_exceptions + def net_connections(self, kind='inet'): + ret = net_connections(kind, _pid=self.pid) + # The underlying C implementation retrieves all OS connections + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not ret: + # will raise NSP if process is gone + os.stat(f"{self._procfs_path}/{self.pid}") + return ret + + @wrap_exceptions + def nice_get(self): + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def ppid(self): + self._ppid = self._proc_basic_info()[proc_info_map['ppid']] + return self._ppid + + @wrap_exceptions + def uids(self): + real, effective, saved, _, _, _ = self._proc_cred() + return _common.puids(real, effective, saved) + + @wrap_exceptions + def gids(self): + _, _, _, real, effective, saved = self._proc_cred() + return _common.puids(real, effective, saved) + + @wrap_exceptions + def cpu_times(self): + t = cext.proc_cpu_times(self.pid, self._procfs_path) + return _common.pcputimes(*t) + + @wrap_exceptions + def terminal(self): + ttydev = self._proc_basic_info()[proc_info_map['ttynr']] + # convert from 64-bit dev_t to 32-bit dev_t and then map the device + ttydev = ((ttydev & 0x0000FFFF00000000) >> 16) | (ttydev & 0xFFFF) + # try to match rdev of /dev/pts/* files ttydev + for dev in glob.glob("/dev/**/*"): + if os.stat(dev).st_rdev == ttydev: + return dev + return None + + @wrap_exceptions + def cwd(self): + procfs_path = self._procfs_path + try: + result = os.readlink(f"{procfs_path}/{self.pid}/cwd") + return result.rstrip('/') + except FileNotFoundError: + os.stat(f"{procfs_path}/{self.pid}") # raise NSP or AD + return "" + + @wrap_exceptions + def memory_info(self): + ret = self._proc_basic_info() + rss = ret[proc_info_map['rss']] * 1024 + vms = ret[proc_info_map['vms']] * 1024 + return pmem(rss, vms) + + memory_full_info = memory_info + + @wrap_exceptions + def status(self): + code = self._proc_basic_info()[proc_info_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + def open_files(self): + # TODO rewrite without using procfiles (stat /proc/pid/fd/* and then + # find matching name of the inode) + p = subprocess.Popen( + ["/usr/bin/procfiles", "-n", str(self.pid)], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + stdout, stderr = p.communicate() + stdout, stderr = ( + x.decode(sys.stdout.encoding) for x in (stdout, stderr) + ) + if "no such process" in stderr.lower(): + raise NoSuchProcess(self.pid, self._name) + procfiles = re.findall(r"(\d+): S_IFREG.*name:(.*)\n", stdout) + retlist = [] + for fd, path in procfiles: + path = path.strip() + if path.startswith("//"): + path = path[1:] + if path.lower() == "cannot be retrieved": + continue + retlist.append(_common.popenfile(path, int(fd))) + return retlist + + @wrap_exceptions + def num_fds(self): + if self.pid == 0: # no /proc/0/fd + return 0 + return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd")) + + @wrap_exceptions + def num_ctx_switches(self): + return _common.pctxsw(*cext.proc_num_ctx_switches(self.pid)) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + if HAS_PROC_IO_COUNTERS: + + @wrap_exceptions + def io_counters(self): + try: + rc, wc, rb, wb = cext.proc_io_counters(self.pid) + except OSError as err: + # if process is terminated, proc_io_counters returns OSError + # instead of NSP + if not pid_exists(self.pid): + raise NoSuchProcess(self.pid, self._name) from err + raise + return _common.pio(rc, wc, rb, wb) diff --git a/lib/python3.12/site-packages/psutil/_psbsd.py b/lib/python3.12/site-packages/psutil/_psbsd.py new file mode 100644 index 0000000000000000000000000000000000000000..13bd926fabf1740e0170b1ada1f4bf53bd3855dd --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_psbsd.py @@ -0,0 +1,971 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""FreeBSD, OpenBSD and NetBSD platforms implementation.""" + +import contextlib +import errno +import functools +import os +from collections import defaultdict +from collections import namedtuple +from xml.etree import ElementTree # noqa: ICN001 + +from . import _common +from . import _psposix +from . import _psutil_bsd as cext +from . import _psutil_posix as cext_posix +from ._common import FREEBSD +from ._common import NETBSD +from ._common import OPENBSD +from ._common import AccessDenied +from ._common import NoSuchProcess +from ._common import ZombieProcess +from ._common import conn_tmap +from ._common import conn_to_ntuple +from ._common import debug +from ._common import memoize +from ._common import memoize_when_activated +from ._common import usage_percent + + +__extra__all__ = [] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +if FREEBSD: + PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SRUN: _common.STATUS_RUNNING, + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SWAIT: _common.STATUS_WAITING, + cext.SLOCK: _common.STATUS_LOCKED, + } +elif OPENBSD: + PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SSTOP: _common.STATUS_STOPPED, + # According to /usr/include/sys/proc.h SZOMB is unused. + # test_zombie_process() shows that SDEAD is the right + # equivalent. Also it appears there's no equivalent of + # psutil.STATUS_DEAD. SDEAD really means STATUS_ZOMBIE. + # cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SDEAD: _common.STATUS_ZOMBIE, + cext.SZOMB: _common.STATUS_ZOMBIE, + # From http://www.eecs.harvard.edu/~margo/cs161/videos/proc.h.txt + # OpenBSD has SRUN and SONPROC: SRUN indicates that a process + # is runnable but *not* yet running, i.e. is on a run queue. + # SONPROC indicates that the process is actually executing on + # a CPU, i.e. it is no longer on a run queue. + # As such we'll map SRUN to STATUS_WAKING and SONPROC to + # STATUS_RUNNING + cext.SRUN: _common.STATUS_WAKING, + cext.SONPROC: _common.STATUS_RUNNING, + } +elif NETBSD: + PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SRUN: _common.STATUS_WAKING, + cext.SONPROC: _common.STATUS_RUNNING, + } + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +PAGESIZE = cext_posix.getpagesize() +AF_LINK = cext_posix.AF_LINK + +HAS_PER_CPU_TIMES = hasattr(cext, "per_cpu_times") +HAS_PROC_NUM_THREADS = hasattr(cext, "proc_num_threads") +HAS_PROC_OPEN_FILES = hasattr(cext, 'proc_open_files') +HAS_PROC_NUM_FDS = hasattr(cext, 'proc_num_fds') + +kinfo_proc_map = dict( + ppid=0, + status=1, + real_uid=2, + effective_uid=3, + saved_uid=4, + real_gid=5, + effective_gid=6, + saved_gid=7, + ttynr=8, + create_time=9, + ctx_switches_vol=10, + ctx_switches_unvol=11, + read_io_count=12, + write_io_count=13, + user_time=14, + sys_time=15, + ch_user_time=16, + ch_sys_time=17, + rss=18, + vms=19, + memtext=20, + memdata=21, + memstack=22, + cpunum=23, + name=24, +) + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# fmt: off +# psutil.virtual_memory() +svmem = namedtuple( + 'svmem', ['total', 'available', 'percent', 'used', 'free', + 'active', 'inactive', 'buffers', 'cached', 'shared', 'wired']) +# psutil.cpu_times() +scputimes = namedtuple( + 'scputimes', ['user', 'nice', 'system', 'idle', 'irq']) +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms', 'text', 'data', 'stack']) +# psutil.Process.memory_full_info() +pfullmem = pmem +# psutil.Process.cpu_times() +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system']) +# psutil.Process.memory_maps(grouped=True) +pmmap_grouped = namedtuple( + 'pmmap_grouped', 'path rss, private, ref_count, shadow_count') +# psutil.Process.memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr, perms path rss, private, ref_count, shadow_count') +# psutil.disk_io_counters() +if FREEBSD: + sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_time', 'write_time', + 'busy_time']) +else: + sdiskio = namedtuple('sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes']) +# fmt: on + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + mem = cext.virtual_mem() + if NETBSD: + total, free, active, inactive, wired, cached = mem + # On NetBSD buffers and shared mem is determined via /proc. + # The C ext set them to 0. + with open('/proc/meminfo', 'rb') as f: + for line in f: + if line.startswith(b'Buffers:'): + buffers = int(line.split()[1]) * 1024 + elif line.startswith(b'MemShared:'): + shared = int(line.split()[1]) * 1024 + # Before avail was calculated as (inactive + cached + free), + # same as zabbix, but it turned out it could exceed total (see + # #2233), so zabbix seems to be wrong. Htop calculates it + # differently, and the used value seem more realistic, so let's + # match htop. + # https://github.com/htop-dev/htop/blob/e7f447b/netbsd/NetBSDProcessList.c#L162 + # https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/netbsd/memory.c#L135 + used = active + wired + avail = total - used + else: + total, free, active, inactive, wired, cached, buffers, shared = mem + # matches freebsd-memory CLI: + # * https://people.freebsd.org/~rse/dist/freebsd-memory + # * https://www.cyberciti.biz/files/scripts/freebsd-memory.pl.txt + # matches zabbix: + # * https://github.com/zabbix/zabbix/blob/af5e0f8/src/libs/zbxsysinfo/freebsd/memory.c#L143 + avail = inactive + cached + free + used = active + wired + cached + + percent = usage_percent((total - avail), total, round_=1) + return svmem( + total, + avail, + percent, + used, + free, + active, + inactive, + buffers, + cached, + shared, + wired, + ) + + +def swap_memory(): + """System swap memory as (total, used, free, sin, sout) namedtuple.""" + total, used, free, sin, sout = cext.swap_mem() + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system per-CPU times as a namedtuple.""" + user, nice, system, idle, irq = cext.cpu_times() + return scputimes(user, nice, system, idle, irq) + + +if HAS_PER_CPU_TIMES: + + def per_cpu_times(): + """Return system CPU times as a namedtuple.""" + ret = [] + for cpu_t in cext.per_cpu_times(): + user, nice, system, idle, irq = cpu_t + item = scputimes(user, nice, system, idle, irq) + ret.append(item) + return ret + +else: + # XXX + # Ok, this is very dirty. + # On FreeBSD < 8 we cannot gather per-cpu information, see: + # https://github.com/giampaolo/psutil/issues/226 + # If num cpus > 1, on first call we return single cpu times to avoid a + # crash at psutil import time. + # Next calls will fail with NotImplementedError + def per_cpu_times(): + """Return system CPU times as a namedtuple.""" + if cpu_count_logical() == 1: + return [cpu_times()] + if per_cpu_times.__called__: + msg = "supported only starting from FreeBSD 8" + raise NotImplementedError(msg) + per_cpu_times.__called__ = True + return [cpu_times()] + + per_cpu_times.__called__ = False + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + return cext.cpu_count_logical() + + +if OPENBSD or NETBSD: + + def cpu_count_cores(): + # OpenBSD and NetBSD do not implement this. + return 1 if cpu_count_logical() == 1 else None + +else: + + def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + # From the C module we'll get an XML string similar to this: + # http://manpages.ubuntu.com/manpages/precise/man4/smp.4freebsd.html + # We may get None in case "sysctl kern.sched.topology_spec" + # is not supported on this BSD version, in which case we'll mimic + # os.cpu_count() and return None. + ret = None + s = cext.cpu_topology() + if s is not None: + # get rid of padding chars appended at the end of the string + index = s.rfind("") + if index != -1: + s = s[: index + 9] + root = ElementTree.fromstring(s) + try: + ret = len(root.findall('group/children/group/cpu')) or None + finally: + # needed otherwise it will memleak + root.clear() + if not ret: + # If logical CPUs == 1 it's obvious we' have only 1 core. + if cpu_count_logical() == 1: + return 1 + return ret + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + if FREEBSD: + # Note: the C ext is returning some metrics we are not exposing: + # traps. + ctxsw, intrs, soft_intrs, syscalls, _traps = cext.cpu_stats() + elif NETBSD: + # XXX + # Note about intrs: the C extension returns 0. intrs + # can be determined via /proc/stat; it has the same value as + # soft_intrs thought so the kernel is faking it (?). + # + # Note about syscalls: the C extension always sets it to 0 (?). + # + # Note: the C ext is returning some metrics we are not exposing: + # traps, faults and forks. + ctxsw, intrs, soft_intrs, syscalls, _traps, _faults, _forks = ( + cext.cpu_stats() + ) + with open('/proc/stat', 'rb') as f: + for line in f: + if line.startswith(b'intr'): + intrs = int(line.split()[1]) + elif OPENBSD: + # Note: the C ext is returning some metrics we are not exposing: + # traps, faults and forks. + ctxsw, intrs, soft_intrs, syscalls, _traps, _faults, _forks = ( + cext.cpu_stats() + ) + return _common.scpustats(ctxsw, intrs, soft_intrs, syscalls) + + +if FREEBSD: + + def cpu_freq(): + """Return frequency metrics for CPUs. As of Dec 2018 only + CPU 0 appears to be supported by FreeBSD and all other cores + match the frequency of CPU 0. + """ + ret = [] + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, available_freq = cext.cpu_freq(cpu) + except NotImplementedError: + continue + if available_freq: + try: + min_freq = int(available_freq.split(" ")[-1].split("/")[0]) + except (IndexError, ValueError): + min_freq = None + try: + max_freq = int(available_freq.split(" ")[0].split("/")[0]) + except (IndexError, ValueError): + max_freq = None + ret.append(_common.scpufreq(current, min_freq, max_freq)) + return ret + +elif OPENBSD: + + def cpu_freq(): + curr = float(cext.cpu_freq()) + return [_common.scpufreq(curr, 0.0, 0.0)] + + +# ===================================================================== +# --- disks +# ===================================================================== + + +def disk_partitions(all=False): + """Return mounted disk partitions as a list of namedtuples. + 'all' argument is ignored, see: + https://github.com/giampaolo/psutil/issues/906. + """ + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +disk_usage = _psposix.disk_usage +disk_io_counters = cext.disk_io_counters + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_io_counters = cext.net_io_counters +net_if_addrs = cext_posix.net_if_addrs + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + names = net_io_counters().keys() + ret = {} + for name in names: + try: + mtu = cext_posix.net_if_mtu(name) + flags = cext_posix.net_if_flags(name) + duplex, speed = cext_posix.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + output_flags = ','.join(flags) + isup = 'running' in flags + ret[name] = _common.snicstats( + isup, duplex, speed, mtu, output_flags + ) + return ret + + +def net_connections(kind): + """System-wide network connections.""" + families, types = conn_tmap[kind] + ret = set() + if OPENBSD: + rawlist = cext.net_connections(-1, families, types) + elif NETBSD: + rawlist = cext.net_connections(-1, kind) + else: # FreeBSD + rawlist = cext.net_connections(families, types) + + for item in rawlist: + fd, fam, type, laddr, raddr, status, pid = item + nt = conn_to_ntuple( + fd, fam, type, laddr, raddr, status, TCP_STATUSES, pid + ) + ret.add(nt) + return list(ret) + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +if FREEBSD: + + def sensors_battery(): + """Return battery info.""" + try: + percent, minsleft, power_plugged = cext.sensors_battery() + except NotImplementedError: + # See: https://github.com/giampaolo/psutil/issues/1074 + return None + power_plugged = power_plugged == 1 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + elif minsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = minsleft * 60 + return _common.sbattery(percent, secsleft, power_plugged) + + def sensors_temperatures(): + """Return CPU cores temperatures if available, else an empty dict.""" + ret = defaultdict(list) + num_cpus = cpu_count_logical() + for cpu in range(num_cpus): + try: + current, high = cext.sensors_cpu_temperature(cpu) + if high <= 0: + high = None + name = f"Core {cpu}" + ret["coretemp"].append( + _common.shwtemp(name, current, high, high) + ) + except NotImplementedError: + pass + + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, tty, hostname, tstamp, pid = item + if pid == -1: + assert OPENBSD + pid = None + if tty == '~': + continue # reboot or shutdown + nt = _common.suser(user, tty or None, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +@memoize +def _pid_0_exists(): + try: + Process(0).name() + except NoSuchProcess: + return False + except AccessDenied: + return True + else: + return True + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + ret = cext.pids() + if OPENBSD and (0 not in ret) and _pid_0_exists(): + # On OpenBSD the kernel does not return PID 0 (neither does + # ps) but it's actually querable (Process(0) will succeed). + ret.insert(0, 0) + return ret + + +if NETBSD: + + def pid_exists(pid): + exists = _psposix.pid_exists(pid) + if not exists: + # We do this because _psposix.pid_exists() lies in case of + # zombie processes. + return pid in pids() + else: + return True + +elif OPENBSD: + + def pid_exists(pid): + exists = _psposix.pid_exists(pid) + if not exists: + return False + else: + # OpenBSD seems to be the only BSD platform where + # _psposix.pid_exists() returns True for thread IDs (tids), + # so we can't use it. + return pid in pids() + +else: # FreeBSD + pid_exists = _psposix.pid_exists + + +def is_zombie(pid): + try: + st = cext.proc_oneshot_info(pid)[kinfo_proc_map['status']] + return PROC_STATUSES.get(st) == _common.STATUS_ZOMBIE + except OSError: + return False + + +def wrap_exceptions(fun): + """Decorator which translates bare OSError exceptions into + NoSuchProcess and AccessDenied. + """ + + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name + try: + return fun(self, *args, **kwargs) + except ProcessLookupError as err: + if is_zombie(pid): + raise ZombieProcess(pid, name, ppid) from err + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + except OSError as err: + if pid == 0 and 0 in pids(): + raise AccessDenied(pid, name) from err + raise + + return wrapper + + +@contextlib.contextmanager +def wrap_exceptions_procfs(inst): + """Same as above, for routines relying on reading /proc fs.""" + pid, name, ppid = inst.pid, inst._name, inst._ppid + try: + yield + except (ProcessLookupError, FileNotFoundError) as err: + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if is_zombie(inst.pid): + raise ZombieProcess(pid, name, ppid) from err + else: + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + + +class Process: + """Wrapper class around underlying C implementation.""" + + __slots__ = ["_cache", "_name", "_ppid", "pid"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + cext.proc_name(self.pid) + + @wrap_exceptions + @memoize_when_activated + def oneshot(self): + """Retrieves multiple process info in one shot as a raw tuple.""" + ret = cext.proc_oneshot_info(self.pid) + assert len(ret) == len(kinfo_proc_map) + return ret + + def oneshot_enter(self): + self.oneshot.cache_activate(self) + + def oneshot_exit(self): + self.oneshot.cache_deactivate(self) + + @wrap_exceptions + def name(self): + name = self.oneshot()[kinfo_proc_map['name']] + return name if name is not None else cext.proc_name(self.pid) + + @wrap_exceptions + def exe(self): + if FREEBSD: + if self.pid == 0: + return '' # else NSP + return cext.proc_exe(self.pid) + elif NETBSD: + if self.pid == 0: + # /proc/0 dir exists but /proc/0/exe doesn't + return "" + with wrap_exceptions_procfs(self): + return os.readlink(f"/proc/{self.pid}/exe") + else: + # OpenBSD: exe cannot be determined; references: + # https://chromium.googlesource.com/chromium/src/base/+/ + # master/base_paths_posix.cc + # We try our best guess by using which against the first + # cmdline arg (may return None). + import shutil + + cmdline = self.cmdline() + if cmdline: + return shutil.which(cmdline[0]) or "" + else: + return "" + + @wrap_exceptions + def cmdline(self): + if OPENBSD and self.pid == 0: + return [] # ...else it crashes + elif NETBSD: + # XXX - most of the times the underlying sysctl() call on + # NetBSD and OpenBSD returns a truncated string. Also + # /proc/pid/cmdline behaves the same so it looks like this + # is a kernel bug. + try: + return cext.proc_cmdline(self.pid) + except OSError as err: + if err.errno == errno.EINVAL: + pid, name, ppid = self.pid, self._name, self._ppid + if is_zombie(self.pid): + raise ZombieProcess(pid, name, ppid) from err + if not pid_exists(self.pid): + raise NoSuchProcess(pid, name, ppid) from err + # XXX: this happens with unicode tests. It means the C + # routine is unable to decode invalid unicode chars. + debug(f"ignoring {err!r} and returning an empty list") + return [] + else: + raise + else: + return cext.proc_cmdline(self.pid) + + @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid) + + @wrap_exceptions + def terminal(self): + tty_nr = self.oneshot()[kinfo_proc_map['ttynr']] + tmap = _psposix.get_terminal_map() + try: + return tmap[tty_nr] + except KeyError: + return None + + @wrap_exceptions + def ppid(self): + self._ppid = self.oneshot()[kinfo_proc_map['ppid']] + return self._ppid + + @wrap_exceptions + def uids(self): + rawtuple = self.oneshot() + return _common.puids( + rawtuple[kinfo_proc_map['real_uid']], + rawtuple[kinfo_proc_map['effective_uid']], + rawtuple[kinfo_proc_map['saved_uid']], + ) + + @wrap_exceptions + def gids(self): + rawtuple = self.oneshot() + return _common.pgids( + rawtuple[kinfo_proc_map['real_gid']], + rawtuple[kinfo_proc_map['effective_gid']], + rawtuple[kinfo_proc_map['saved_gid']], + ) + + @wrap_exceptions + def cpu_times(self): + rawtuple = self.oneshot() + return _common.pcputimes( + rawtuple[kinfo_proc_map['user_time']], + rawtuple[kinfo_proc_map['sys_time']], + rawtuple[kinfo_proc_map['ch_user_time']], + rawtuple[kinfo_proc_map['ch_sys_time']], + ) + + if FREEBSD: + + @wrap_exceptions + def cpu_num(self): + return self.oneshot()[kinfo_proc_map['cpunum']] + + @wrap_exceptions + def memory_info(self): + rawtuple = self.oneshot() + return pmem( + rawtuple[kinfo_proc_map['rss']], + rawtuple[kinfo_proc_map['vms']], + rawtuple[kinfo_proc_map['memtext']], + rawtuple[kinfo_proc_map['memdata']], + rawtuple[kinfo_proc_map['memstack']], + ) + + memory_full_info = memory_info + + @wrap_exceptions + def create_time(self): + return self.oneshot()[kinfo_proc_map['create_time']] + + @wrap_exceptions + def num_threads(self): + if HAS_PROC_NUM_THREADS: + # FreeBSD + return cext.proc_num_threads(self.pid) + else: + return len(self.threads()) + + @wrap_exceptions + def num_ctx_switches(self): + rawtuple = self.oneshot() + return _common.pctxsw( + rawtuple[kinfo_proc_map['ctx_switches_vol']], + rawtuple[kinfo_proc_map['ctx_switches_unvol']], + ) + + @wrap_exceptions + def threads(self): + # Note: on OpenSBD this (/dev/mem) requires root access. + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + if OPENBSD: + self._assert_alive() + return retlist + + @wrap_exceptions + def net_connections(self, kind='inet'): + families, types = conn_tmap[kind] + ret = [] + + if NETBSD: + rawlist = cext.net_connections(self.pid, kind) + elif OPENBSD: + rawlist = cext.net_connections(self.pid, families, types) + else: + rawlist = cext.proc_net_connections(self.pid, families, types) + + for item in rawlist: + fd, fam, type, laddr, raddr, status = item[:6] + if FREEBSD: + if (fam not in families) or (type not in types): + continue + nt = conn_to_ntuple( + fd, fam, type, laddr, raddr, status, TCP_STATUSES + ) + ret.append(nt) + + self._assert_alive() + return ret + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + @wrap_exceptions + def nice_get(self): + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def status(self): + code = self.oneshot()[kinfo_proc_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + @wrap_exceptions + def io_counters(self): + rawtuple = self.oneshot() + return _common.pio( + rawtuple[kinfo_proc_map['read_io_count']], + rawtuple[kinfo_proc_map['write_io_count']], + -1, + -1, + ) + + @wrap_exceptions + def cwd(self): + """Return process current working directory.""" + # sometimes we get an empty string, in which case we turn + # it into None + if OPENBSD and self.pid == 0: + return "" # ...else it would raise EINVAL + elif NETBSD or HAS_PROC_OPEN_FILES: + # FreeBSD < 8 does not support functions based on + # kinfo_getfile() and kinfo_getvmmap() + return cext.proc_cwd(self.pid) + else: + raise NotImplementedError( + "supported only starting from FreeBSD 8" if FREEBSD else "" + ) + + nt_mmap_grouped = namedtuple( + 'mmap', 'path rss, private, ref_count, shadow_count' + ) + nt_mmap_ext = namedtuple( + 'mmap', 'addr, perms path rss, private, ref_count, shadow_count' + ) + + def _not_implemented(self): + raise NotImplementedError + + # FreeBSD < 8 does not support functions based on kinfo_getfile() + # and kinfo_getvmmap() + if HAS_PROC_OPEN_FILES: + + @wrap_exceptions + def open_files(self): + """Return files opened by process as a list of namedtuples.""" + rawlist = cext.proc_open_files(self.pid) + return [_common.popenfile(path, fd) for path, fd in rawlist] + + else: + open_files = _not_implemented + + # FreeBSD < 8 does not support functions based on kinfo_getfile() + # and kinfo_getvmmap() + if HAS_PROC_NUM_FDS: + + @wrap_exceptions + def num_fds(self): + """Return the number of file descriptors opened by this process.""" + ret = cext.proc_num_fds(self.pid) + if NETBSD: + self._assert_alive() + return ret + + else: + num_fds = _not_implemented + + # --- FreeBSD only APIs + + if FREEBSD: + + @wrap_exceptions + def cpu_affinity_get(self): + return cext.proc_cpu_affinity_get(self.pid) + + @wrap_exceptions + def cpu_affinity_set(self, cpus): + # Pre-emptively check if CPUs are valid because the C + # function has a weird behavior in case of invalid CPUs, + # see: https://github.com/giampaolo/psutil/issues/586 + allcpus = set(range(len(per_cpu_times()))) + for cpu in cpus: + if cpu not in allcpus: + msg = f"invalid CPU {cpu!r} (choose between {allcpus})" + raise ValueError(msg) + try: + cext.proc_cpu_affinity_set(self.pid, cpus) + except OSError as err: + # 'man cpuset_setaffinity' about EDEADLK: + # <> + if err.errno in {errno.EINVAL, errno.EDEADLK}: + for cpu in cpus: + if cpu not in allcpus: + msg = ( + f"invalid CPU {cpu!r} (choose between" + f" {allcpus})" + ) + raise ValueError(msg) from err + raise + + @wrap_exceptions + def memory_maps(self): + return cext.proc_memory_maps(self.pid) + + @wrap_exceptions + def rlimit(self, resource, limits=None): + if limits is None: + return cext.proc_getrlimit(self.pid, resource) + else: + if len(limits) != 2: + msg = ( + "second argument must be a (soft, hard) tuple, got" + f" {limits!r}" + ) + raise ValueError(msg) + soft, hard = limits + return cext.proc_setrlimit(self.pid, resource, soft, hard) diff --git a/lib/python3.12/site-packages/psutil/_pslinux.py b/lib/python3.12/site-packages/psutil/_pslinux.py new file mode 100644 index 0000000000000000000000000000000000000000..8cc64e9a1000bddf5dde795868dcdcb782c35a1c --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_pslinux.py @@ -0,0 +1,2295 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Linux platform implementation.""" + + +import base64 +import collections +import enum +import errno +import functools +import glob +import os +import re +import resource +import socket +import struct +import sys +import warnings +from collections import defaultdict +from collections import namedtuple + +from . import _common +from . import _psposix +from . import _psutil_linux as cext +from . import _psutil_posix as cext_posix +from ._common import ENCODING +from ._common import NIC_DUPLEX_FULL +from ._common import NIC_DUPLEX_HALF +from ._common import NIC_DUPLEX_UNKNOWN +from ._common import AccessDenied +from ._common import NoSuchProcess +from ._common import ZombieProcess +from ._common import bcat +from ._common import cat +from ._common import debug +from ._common import decode +from ._common import get_procfs_path +from ._common import isfile_strict +from ._common import memoize +from ._common import memoize_when_activated +from ._common import open_binary +from ._common import open_text +from ._common import parse_environ_block +from ._common import path_exists_strict +from ._common import supports_ipv6 +from ._common import usage_percent + + +# fmt: off +__extra__all__ = [ + 'PROCFS_PATH', + # io prio constants + "IOPRIO_CLASS_NONE", "IOPRIO_CLASS_RT", "IOPRIO_CLASS_BE", + "IOPRIO_CLASS_IDLE", + # connection status constants + "CONN_ESTABLISHED", "CONN_SYN_SENT", "CONN_SYN_RECV", "CONN_FIN_WAIT1", + "CONN_FIN_WAIT2", "CONN_TIME_WAIT", "CONN_CLOSE", "CONN_CLOSE_WAIT", + "CONN_LAST_ACK", "CONN_LISTEN", "CONN_CLOSING", +] + +if hasattr(resource, "prlimit"): + __extra__all__.extend( + [x for x in dir(cext) if x.startswith('RLIM') and x.isupper()] + ) +# fmt: on + + +# ===================================================================== +# --- globals +# ===================================================================== + + +POWER_SUPPLY_PATH = "/sys/class/power_supply" +HAS_PROC_SMAPS = os.path.exists(f"/proc/{os.getpid()}/smaps") +HAS_PROC_SMAPS_ROLLUP = os.path.exists(f"/proc/{os.getpid()}/smaps_rollup") +HAS_PROC_IO_PRIORITY = hasattr(cext, "proc_ioprio_get") +HAS_CPU_AFFINITY = hasattr(cext, "proc_cpu_affinity_get") + +# Number of clock ticks per second +CLOCK_TICKS = os.sysconf("SC_CLK_TCK") +PAGESIZE = cext_posix.getpagesize() +BOOT_TIME = None # set later +LITTLE_ENDIAN = sys.byteorder == 'little' + +# "man iostat" states that sectors are equivalent with blocks and have +# a size of 512 bytes. Despite this value can be queried at runtime +# via /sys/block/{DISK}/queue/hw_sector_size and results may vary +# between 1k, 2k, or 4k... 512 appears to be a magic constant used +# throughout Linux source code: +# * https://stackoverflow.com/a/38136179/376587 +# * https://lists.gt.net/linux/kernel/2241060 +# * https://github.com/giampaolo/psutil/issues/1305 +# * https://github.com/torvalds/linux/blob/ +# 4f671fe2f9523a1ea206f63fe60a7c7b3a56d5c7/include/linux/bio.h#L99 +# * https://lkml.org/lkml/2015/8/17/234 +DISK_SECTOR_SIZE = 512 + +AddressFamily = enum.IntEnum( + 'AddressFamily', {'AF_LINK': int(socket.AF_PACKET)} +) +AF_LINK = AddressFamily.AF_LINK + + +# ioprio_* constants http://linux.die.net/man/2/ioprio_get +class IOPriority(enum.IntEnum): + IOPRIO_CLASS_NONE = 0 + IOPRIO_CLASS_RT = 1 + IOPRIO_CLASS_BE = 2 + IOPRIO_CLASS_IDLE = 3 + + +globals().update(IOPriority.__members__) + +# See: +# https://github.com/torvalds/linux/blame/master/fs/proc/array.c +# ...and (TASK_* constants): +# https://github.com/torvalds/linux/blob/master/include/linux/sched.h +PROC_STATUSES = { + "R": _common.STATUS_RUNNING, + "S": _common.STATUS_SLEEPING, + "D": _common.STATUS_DISK_SLEEP, + "T": _common.STATUS_STOPPED, + "t": _common.STATUS_TRACING_STOP, + "Z": _common.STATUS_ZOMBIE, + "X": _common.STATUS_DEAD, + "x": _common.STATUS_DEAD, + "K": _common.STATUS_WAKE_KILL, + "W": _common.STATUS_WAKING, + "I": _common.STATUS_IDLE, + "P": _common.STATUS_PARKED, +} + +# https://github.com/torvalds/linux/blob/master/include/net/tcp_states.h +TCP_STATUSES = { + "01": _common.CONN_ESTABLISHED, + "02": _common.CONN_SYN_SENT, + "03": _common.CONN_SYN_RECV, + "04": _common.CONN_FIN_WAIT1, + "05": _common.CONN_FIN_WAIT2, + "06": _common.CONN_TIME_WAIT, + "07": _common.CONN_CLOSE, + "08": _common.CONN_CLOSE_WAIT, + "09": _common.CONN_LAST_ACK, + "0A": _common.CONN_LISTEN, + "0B": _common.CONN_CLOSING, +} + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# fmt: off +# psutil.virtual_memory() +svmem = namedtuple( + 'svmem', ['total', 'available', 'percent', 'used', 'free', + 'active', 'inactive', 'buffers', 'cached', 'shared', 'slab']) +# psutil.disk_io_counters() +sdiskio = namedtuple( + 'sdiskio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_time', 'write_time', + 'read_merged_count', 'write_merged_count', + 'busy_time']) +# psutil.Process().open_files() +popenfile = namedtuple( + 'popenfile', ['path', 'fd', 'position', 'mode', 'flags']) +# psutil.Process().memory_info() +pmem = namedtuple('pmem', 'rss vms shared text lib data dirty') +# psutil.Process().memory_full_info() +pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', 'pss', 'swap')) +# psutil.Process().memory_maps(grouped=True) +pmmap_grouped = namedtuple( + 'pmmap_grouped', + ['path', 'rss', 'size', 'pss', 'shared_clean', 'shared_dirty', + 'private_clean', 'private_dirty', 'referenced', 'anonymous', 'swap']) +# psutil.Process().memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) +# psutil.Process.io_counters() +pio = namedtuple('pio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'read_chars', 'write_chars']) +# psutil.Process.cpu_times() +pcputimes = namedtuple('pcputimes', + ['user', 'system', 'children_user', 'children_system', + 'iowait']) +# fmt: on + + +# ===================================================================== +# --- utils +# ===================================================================== + + +def readlink(path): + """Wrapper around os.readlink().""" + assert isinstance(path, str), path + path = os.readlink(path) + # readlink() might return paths containing null bytes ('\x00') + # resulting in "TypeError: must be encoded string without NULL + # bytes, not str" errors when the string is passed to other + # fs-related functions (os.*, open(), ...). + # Apparently everything after '\x00' is garbage (we can have + # ' (deleted)', 'new' and possibly others), see: + # https://github.com/giampaolo/psutil/issues/717 + path = path.split('\x00')[0] + # Certain paths have ' (deleted)' appended. Usually this is + # bogus as the file actually exists. Even if it doesn't we + # don't care. + if path.endswith(' (deleted)') and not path_exists_strict(path): + path = path[:-10] + return path + + +def file_flags_to_mode(flags): + """Convert file's open() flags into a readable string. + Used by Process.open_files(). + """ + modes_map = {os.O_RDONLY: 'r', os.O_WRONLY: 'w', os.O_RDWR: 'w+'} + mode = modes_map[flags & (os.O_RDONLY | os.O_WRONLY | os.O_RDWR)] + if flags & os.O_APPEND: + mode = mode.replace('w', 'a', 1) + mode = mode.replace('w+', 'r+') + # possible values: r, w, a, r+, a+ + return mode + + +def is_storage_device(name): + """Return True if the given name refers to a root device (e.g. + "sda", "nvme0n1") as opposed to a logical partition (e.g. "sda1", + "nvme0n1p1"). If name is a virtual device (e.g. "loop1", "ram") + return True. + """ + # Re-adapted from iostat source code, see: + # https://github.com/sysstat/sysstat/blob/ + # 97912938cd476645b267280069e83b1c8dc0e1c7/common.c#L208 + # Some devices may have a slash in their name (e.g. cciss/c0d0...). + name = name.replace('/', '!') + including_virtual = True + if including_virtual: + path = f"/sys/block/{name}" + else: + path = f"/sys/block/{name}/device" + return os.access(path, os.F_OK) + + +@memoize +def set_scputimes_ntuple(procfs_path): + """Set a namedtuple of variable fields depending on the CPU times + available on this Linux kernel version which may be: + (user, nice, system, idle, iowait, irq, softirq, [steal, [guest, + [guest_nice]]]) + Used by cpu_times() function. + """ + global scputimes + with open_binary(f"{procfs_path}/stat") as f: + values = f.readline().split()[1:] + fields = ['user', 'nice', 'system', 'idle', 'iowait', 'irq', 'softirq'] + vlen = len(values) + if vlen >= 8: + # Linux >= 2.6.11 + fields.append('steal') + if vlen >= 9: + # Linux >= 2.6.24 + fields.append('guest') + if vlen >= 10: + # Linux >= 3.2.0 + fields.append('guest_nice') + scputimes = namedtuple('scputimes', fields) + + +try: + set_scputimes_ntuple("/proc") +except Exception as err: # noqa: BLE001 + # Don't want to crash at import time. + debug(f"ignoring exception on import: {err!r}") + scputimes = namedtuple('scputimes', 'user system idle')(0.0, 0.0, 0.0) + + +# ===================================================================== +# --- system memory +# ===================================================================== + + +def calculate_avail_vmem(mems): + """Fallback for kernels < 3.14 where /proc/meminfo does not provide + "MemAvailable", see: + https://blog.famzah.net/2014/09/24/. + + This code reimplements the algorithm outlined here: + https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ + commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + + We use this function also when "MemAvailable" returns 0 (possibly a + kernel bug, see: https://github.com/giampaolo/psutil/issues/1915). + In that case this routine matches "free" CLI tool result ("available" + column). + + XXX: on recent kernels this calculation may differ by ~1.5% compared + to "MemAvailable:", as it's calculated slightly differently. + It is still way more realistic than doing (free + cached) though. + See: + * https://gitlab.com/procps-ng/procps/issues/42 + * https://github.com/famzah/linux-memavailable-procfs/issues/2 + """ + # Note about "fallback" value. According to: + # https://git.kernel.org/cgit/linux/kernel/git/torvalds/linux.git/ + # commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773 + # ...long ago "available" memory was calculated as (free + cached), + # We use fallback when one of these is missing from /proc/meminfo: + # "Active(file)": introduced in 2.6.28 / Dec 2008 + # "Inactive(file)": introduced in 2.6.28 / Dec 2008 + # "SReclaimable": introduced in 2.6.19 / Nov 2006 + # /proc/zoneinfo: introduced in 2.6.13 / Aug 2005 + free = mems[b'MemFree:'] + fallback = free + mems.get(b"Cached:", 0) + try: + lru_active_file = mems[b'Active(file):'] + lru_inactive_file = mems[b'Inactive(file):'] + slab_reclaimable = mems[b'SReclaimable:'] + except KeyError as err: + debug( + f"{err.args[0]} is missing from /proc/meminfo; using an" + " approximation for calculating available memory" + ) + return fallback + try: + f = open_binary(f"{get_procfs_path()}/zoneinfo") + except OSError: + return fallback # kernel 2.6.13 + + watermark_low = 0 + with f: + for line in f: + line = line.strip() + if line.startswith(b'low'): + watermark_low += int(line.split()[1]) + watermark_low *= PAGESIZE + + avail = free - watermark_low + pagecache = lru_active_file + lru_inactive_file + pagecache -= min(pagecache / 2, watermark_low) + avail += pagecache + avail += slab_reclaimable - min(slab_reclaimable / 2.0, watermark_low) + return int(avail) + + +def virtual_memory(): + """Report virtual memory stats. + This implementation mimics procps-ng-3.3.12, aka "free" CLI tool: + https://gitlab.com/procps-ng/procps/blob/ + 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L778-791 + The returned values are supposed to match both "free" and "vmstat -s" + CLI tools. + """ + missing_fields = [] + mems = {} + with open_binary(f"{get_procfs_path()}/meminfo") as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + + # /proc doc states that the available fields in /proc/meminfo vary + # by architecture and compile options, but these 3 values are also + # returned by sysinfo(2); as such we assume they are always there. + total = mems[b'MemTotal:'] + free = mems[b'MemFree:'] + try: + buffers = mems[b'Buffers:'] + except KeyError: + # https://github.com/giampaolo/psutil/issues/1010 + buffers = 0 + missing_fields.append('buffers') + try: + cached = mems[b"Cached:"] + except KeyError: + cached = 0 + missing_fields.append('cached') + else: + # "free" cmdline utility sums reclaimable to cached. + # Older versions of procps used to add slab memory instead. + # This got changed in: + # https://gitlab.com/procps-ng/procps/commit/ + # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + cached += mems.get(b"SReclaimable:", 0) # since kernel 2.6.19 + + try: + shared = mems[b'Shmem:'] # since kernel 2.6.32 + except KeyError: + try: + shared = mems[b'MemShared:'] # kernels 2.4 + except KeyError: + shared = 0 + missing_fields.append('shared') + + try: + active = mems[b"Active:"] + except KeyError: + active = 0 + missing_fields.append('active') + + try: + inactive = mems[b"Inactive:"] + except KeyError: + try: + inactive = ( + mems[b"Inact_dirty:"] + + mems[b"Inact_clean:"] + + mems[b"Inact_laundry:"] + ) + except KeyError: + inactive = 0 + missing_fields.append('inactive') + + try: + slab = mems[b"Slab:"] + except KeyError: + slab = 0 + + used = total - free - cached - buffers + if used < 0: + # May be symptomatic of running within a LCX container where such + # values will be dramatically distorted over those of the host. + used = total - free + + # - starting from 4.4.0 we match free's "available" column. + # Before 4.4.0 we calculated it as (free + buffers + cached) + # which matched htop. + # - free and htop available memory differs as per: + # http://askubuntu.com/a/369589 + # http://unix.stackexchange.com/a/65852/168884 + # - MemAvailable has been introduced in kernel 3.14 + try: + avail = mems[b'MemAvailable:'] + except KeyError: + avail = calculate_avail_vmem(mems) + else: + if avail == 0: + # Yes, it can happen (probably a kernel bug): + # https://github.com/giampaolo/psutil/issues/1915 + # In this case "free" CLI tool makes an estimate. We do the same, + # and it matches "free" CLI tool. + avail = calculate_avail_vmem(mems) + + if avail < 0: + avail = 0 + missing_fields.append('available') + elif avail > total: + # If avail is greater than total or our calculation overflows, + # that's symptomatic of running within a LCX container where such + # values will be dramatically distorted over those of the host. + # https://gitlab.com/procps-ng/procps/blob/ + # 24fd2605c51fccc375ab0287cec33aa767f06718/proc/sysinfo.c#L764 + avail = free + + percent = usage_percent((total - avail), total, round_=1) + + # Warn about missing metrics which are set to 0. + if missing_fields: + msg = "{} memory stats couldn't be determined and {} set to 0".format( + ", ".join(missing_fields), + "was" if len(missing_fields) == 1 else "were", + ) + warnings.warn(msg, RuntimeWarning, stacklevel=2) + + return svmem( + total, + avail, + percent, + used, + free, + active, + inactive, + buffers, + cached, + shared, + slab, + ) + + +def swap_memory(): + """Return swap memory metrics.""" + mems = {} + with open_binary(f"{get_procfs_path()}/meminfo") as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + # We prefer /proc/meminfo over sysinfo() syscall so that + # psutil.PROCFS_PATH can be used in order to allow retrieval + # for linux containers, see: + # https://github.com/giampaolo/psutil/issues/1015 + try: + total = mems[b'SwapTotal:'] + free = mems[b'SwapFree:'] + except KeyError: + _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() + total *= unit_multiplier + free *= unit_multiplier + + used = total - free + percent = usage_percent(used, total, round_=1) + # get pgin/pgouts + try: + f = open_binary(f"{get_procfs_path()}/vmstat") + except OSError as err: + # see https://github.com/giampaolo/psutil/issues/722 + msg = ( + "'sin' and 'sout' swap memory stats couldn't " + f"be determined and were set to 0 ({err})" + ) + warnings.warn(msg, RuntimeWarning, stacklevel=2) + sin = sout = 0 + else: + with f: + sin = sout = None + for line in f: + # values are expressed in 4 kilo bytes, we want + # bytes instead + if line.startswith(b'pswpin'): + sin = int(line.split(b' ')[1]) * 4 * 1024 + elif line.startswith(b'pswpout'): + sout = int(line.split(b' ')[1]) * 4 * 1024 + if sin is not None and sout is not None: + break + else: + # we might get here when dealing with exotic Linux + # flavors, see: + # https://github.com/giampaolo/psutil/issues/313 + msg = "'sin' and 'sout' swap memory stats couldn't " + msg += "be determined and were set to 0" + warnings.warn(msg, RuntimeWarning, stacklevel=2) + sin = sout = 0 + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return a named tuple representing the following system-wide + CPU times: + (user, nice, system, idle, iowait, irq, softirq [steal, [guest, + [guest_nice]]]) + Last 3 fields may not be available on all Linux kernel versions. + """ + procfs_path = get_procfs_path() + set_scputimes_ntuple(procfs_path) + with open_binary(f"{procfs_path}/stat") as f: + values = f.readline().split() + fields = values[1 : len(scputimes._fields) + 1] + fields = [float(x) / CLOCK_TICKS for x in fields] + return scputimes(*fields) + + +def per_cpu_times(): + """Return a list of namedtuple representing the CPU times + for every CPU available on the system. + """ + procfs_path = get_procfs_path() + set_scputimes_ntuple(procfs_path) + cpus = [] + with open_binary(f"{procfs_path}/stat") as f: + # get rid of the first line which refers to system wide CPU stats + f.readline() + for line in f: + if line.startswith(b'cpu'): + values = line.split() + fields = values[1 : len(scputimes._fields) + 1] + fields = [float(x) / CLOCK_TICKS for x in fields] + entry = scputimes(*fields) + cpus.append(entry) + return cpus + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + try: + return os.sysconf("SC_NPROCESSORS_ONLN") + except ValueError: + # as a second fallback we try to parse /proc/cpuinfo + num = 0 + with open_binary(f"{get_procfs_path()}/cpuinfo") as f: + for line in f: + if line.lower().startswith(b'processor'): + num += 1 + + # unknown format (e.g. amrel/sparc architectures), see: + # https://github.com/giampaolo/psutil/issues/200 + # try to parse /proc/stat as a last resort + if num == 0: + search = re.compile(r'cpu\d') + with open_text(f"{get_procfs_path()}/stat") as f: + for line in f: + line = line.split(' ')[0] + if search.match(line): + num += 1 + + if num == 0: + # mimic os.cpu_count() + return None + return num + + +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + # Method #1 + ls = set() + # These 2 files are the same but */core_cpus_list is newer while + # */thread_siblings_list is deprecated and may disappear in the future. + # https://www.kernel.org/doc/Documentation/admin-guide/cputopology.rst + # https://github.com/giampaolo/psutil/pull/1727#issuecomment-707624964 + # https://lkml.org/lkml/2019/2/26/41 + p1 = "/sys/devices/system/cpu/cpu[0-9]*/topology/core_cpus_list" + p2 = "/sys/devices/system/cpu/cpu[0-9]*/topology/thread_siblings_list" + for path in glob.glob(p1) or glob.glob(p2): + with open_binary(path) as f: + ls.add(f.read().strip()) + result = len(ls) + if result != 0: + return result + + # Method #2 + mapping = {} + current_info = {} + with open_binary(f"{get_procfs_path()}/cpuinfo") as f: + for line in f: + line = line.strip().lower() + if not line: + # new section + try: + mapping[current_info[b'physical id']] = current_info[ + b'cpu cores' + ] + except KeyError: + pass + current_info = {} + elif line.startswith((b'physical id', b'cpu cores')): + # ongoing section + key, value = line.split(b'\t:', 1) + current_info[key] = int(value) + + result = sum(mapping.values()) + return result or None # mimic os.cpu_count() + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + with open_binary(f"{get_procfs_path()}/stat") as f: + ctx_switches = None + interrupts = None + soft_interrupts = None + for line in f: + if line.startswith(b'ctxt'): + ctx_switches = int(line.split()[1]) + elif line.startswith(b'intr'): + interrupts = int(line.split()[1]) + elif line.startswith(b'softirq'): + soft_interrupts = int(line.split()[1]) + if ( + ctx_switches is not None + and soft_interrupts is not None + and interrupts is not None + ): + break + syscalls = 0 + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls + ) + + +def _cpu_get_cpuinfo_freq(): + """Return current CPU frequency from cpuinfo if available.""" + with open_binary(f"{get_procfs_path()}/cpuinfo") as f: + return [ + float(line.split(b':', 1)[1]) + for line in f + if line.lower().startswith(b'cpu mhz') + ] + + +if os.path.exists("/sys/devices/system/cpu/cpufreq/policy0") or os.path.exists( + "/sys/devices/system/cpu/cpu0/cpufreq" +): + + def cpu_freq(): + """Return frequency metrics for all CPUs. + Contrarily to other OSes, Linux updates these values in + real-time. + """ + cpuinfo_freqs = _cpu_get_cpuinfo_freq() + paths = glob.glob( + "/sys/devices/system/cpu/cpufreq/policy[0-9]*" + ) or glob.glob("/sys/devices/system/cpu/cpu[0-9]*/cpufreq") + paths.sort(key=lambda x: int(re.search(r"[0-9]+", x).group())) + ret = [] + pjoin = os.path.join + for i, path in enumerate(paths): + if len(paths) == len(cpuinfo_freqs): + # take cached value from cpuinfo if available, see: + # https://github.com/giampaolo/psutil/issues/1851 + curr = cpuinfo_freqs[i] * 1000 + else: + curr = bcat(pjoin(path, "scaling_cur_freq"), fallback=None) + if curr is None: + # Likely an old RedHat, see: + # https://github.com/giampaolo/psutil/issues/1071 + curr = bcat(pjoin(path, "cpuinfo_cur_freq"), fallback=None) + if curr is None: + online_path = f"/sys/devices/system/cpu/cpu{i}/online" + # if cpu core is offline, set to all zeroes + if cat(online_path, fallback=None) == "0\n": + ret.append(_common.scpufreq(0.0, 0.0, 0.0)) + continue + msg = "can't find current frequency file" + raise NotImplementedError(msg) + curr = int(curr) / 1000 + max_ = int(bcat(pjoin(path, "scaling_max_freq"))) / 1000 + min_ = int(bcat(pjoin(path, "scaling_min_freq"))) / 1000 + ret.append(_common.scpufreq(curr, min_, max_)) + return ret + +else: + + def cpu_freq(): + """Alternate implementation using /proc/cpuinfo. + min and max frequencies are not available and are set to None. + """ + return [_common.scpufreq(x, 0.0, 0.0) for x in _cpu_get_cpuinfo_freq()] + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_if_addrs = cext_posix.net_if_addrs + + +class _Ipv6UnsupportedError(Exception): + pass + + +class NetConnections: + """A wrapper on top of /proc/net/* files, retrieving per-process + and system-wide open connections (TCP, UDP, UNIX) similarly to + "netstat -an". + + Note: in case of UNIX sockets we're only able to determine the + local endpoint/path, not the one it's connected to. + According to [1] it would be possible but not easily. + + [1] http://serverfault.com/a/417946 + """ + + def __init__(self): + # The string represents the basename of the corresponding + # /proc/net/{proto_name} file. + tcp4 = ("tcp", socket.AF_INET, socket.SOCK_STREAM) + tcp6 = ("tcp6", socket.AF_INET6, socket.SOCK_STREAM) + udp4 = ("udp", socket.AF_INET, socket.SOCK_DGRAM) + udp6 = ("udp6", socket.AF_INET6, socket.SOCK_DGRAM) + unix = ("unix", socket.AF_UNIX, None) + self.tmap = { + "all": (tcp4, tcp6, udp4, udp6, unix), + "tcp": (tcp4, tcp6), + "tcp4": (tcp4,), + "tcp6": (tcp6,), + "udp": (udp4, udp6), + "udp4": (udp4,), + "udp6": (udp6,), + "unix": (unix,), + "inet": (tcp4, tcp6, udp4, udp6), + "inet4": (tcp4, udp4), + "inet6": (tcp6, udp6), + } + self._procfs_path = None + + def get_proc_inodes(self, pid): + inodes = defaultdict(list) + for fd in os.listdir(f"{self._procfs_path}/{pid}/fd"): + try: + inode = readlink(f"{self._procfs_path}/{pid}/fd/{fd}") + except (FileNotFoundError, ProcessLookupError): + # ENOENT == file which is gone in the meantime; + # os.stat(f"/proc/{self.pid}") will be done later + # to force NSP (if it's the case) + continue + except OSError as err: + if err.errno == errno.EINVAL: + # not a link + continue + if err.errno == errno.ENAMETOOLONG: + # file name too long + debug(err) + continue + raise + else: + if inode.startswith('socket:['): + # the process is using a socket + inode = inode[8:][:-1] + inodes[inode].append((pid, int(fd))) + return inodes + + def get_all_inodes(self): + inodes = {} + for pid in pids(): + try: + inodes.update(self.get_proc_inodes(pid)) + except (FileNotFoundError, ProcessLookupError, PermissionError): + # os.listdir() is gonna raise a lot of access denied + # exceptions in case of unprivileged user; that's fine + # as we'll just end up returning a connection with PID + # and fd set to None anyway. + # Both netstat -an and lsof does the same so it's + # unlikely we can do any better. + # ENOENT just means a PID disappeared on us. + continue + return inodes + + @staticmethod + def decode_address(addr, family): + """Accept an "ip:port" address as displayed in /proc/net/* + and convert it into a human readable form, like: + + "0500000A:0016" -> ("10.0.0.5", 22) + "0000000000000000FFFF00000100007F:9E49" -> ("::ffff:127.0.0.1", 40521) + + The IP address portion is a little or big endian four-byte + hexadecimal number; that is, the least significant byte is listed + first, so we need to reverse the order of the bytes to convert it + to an IP address. + The port is represented as a two-byte hexadecimal number. + + Reference: + http://linuxdevcenter.com/pub/a/linux/2000/11/16/LinuxAdmin.html + """ + ip, port = addr.split(':') + port = int(port, 16) + # this usually refers to a local socket in listen mode with + # no end-points connected + if not port: + return () + ip = ip.encode('ascii') + if family == socket.AF_INET: + # see: https://github.com/giampaolo/psutil/issues/201 + if LITTLE_ENDIAN: + ip = socket.inet_ntop(family, base64.b16decode(ip)[::-1]) + else: + ip = socket.inet_ntop(family, base64.b16decode(ip)) + else: # IPv6 + ip = base64.b16decode(ip) + try: + # see: https://github.com/giampaolo/psutil/issues/201 + if LITTLE_ENDIAN: + ip = socket.inet_ntop( + socket.AF_INET6, + struct.pack('>4I', *struct.unpack('<4I', ip)), + ) + else: + ip = socket.inet_ntop( + socket.AF_INET6, + struct.pack('<4I', *struct.unpack('<4I', ip)), + ) + except ValueError: + # see: https://github.com/giampaolo/psutil/issues/623 + if not supports_ipv6(): + raise _Ipv6UnsupportedError from None + raise + return _common.addr(ip, port) + + @staticmethod + def process_inet(file, family, type_, inodes, filter_pid=None): + """Parse /proc/net/tcp* and /proc/net/udp* files.""" + if file.endswith('6') and not os.path.exists(file): + # IPv6 not supported + return + with open_text(file) as f: + f.readline() # skip the first line + for lineno, line in enumerate(f, 1): + try: + _, laddr, raddr, status, _, _, _, _, _, inode = ( + line.split()[:10] + ) + except ValueError: + msg = ( + f"error while parsing {file}; malformed line" + f" {lineno} {line!r}" + ) + raise RuntimeError(msg) from None + if inode in inodes: + # # We assume inet sockets are unique, so we error + # # out if there are multiple references to the + # # same inode. We won't do this for UNIX sockets. + # if len(inodes[inode]) > 1 and family != socket.AF_UNIX: + # raise ValueError("ambiguous inode with multiple " + # "PIDs references") + pid, fd = inodes[inode][0] + else: + pid, fd = None, -1 + if filter_pid is not None and filter_pid != pid: + continue + else: + if type_ == socket.SOCK_STREAM: + status = TCP_STATUSES[status] + else: + status = _common.CONN_NONE + try: + laddr = NetConnections.decode_address(laddr, family) + raddr = NetConnections.decode_address(raddr, family) + except _Ipv6UnsupportedError: + continue + yield (fd, family, type_, laddr, raddr, status, pid) + + @staticmethod + def process_unix(file, family, inodes, filter_pid=None): + """Parse /proc/net/unix files.""" + with open_text(file) as f: + f.readline() # skip the first line + for line in f: + tokens = line.split() + try: + _, _, _, _, type_, _, inode = tokens[0:7] + except ValueError: + if ' ' not in line: + # see: https://github.com/giampaolo/psutil/issues/766 + continue + msg = ( + f"error while parsing {file}; malformed line {line!r}" + ) + raise RuntimeError(msg) # noqa: B904 + if inode in inodes: # noqa: SIM108 + # With UNIX sockets we can have a single inode + # referencing many file descriptors. + pairs = inodes[inode] + else: + pairs = [(None, -1)] + for pid, fd in pairs: + if filter_pid is not None and filter_pid != pid: + continue + else: + path = tokens[-1] if len(tokens) == 8 else '' + type_ = _common.socktype_to_enum(int(type_)) + # XXX: determining the remote endpoint of a + # UNIX socket on Linux is not possible, see: + # https://serverfault.com/questions/252723/ + raddr = "" + status = _common.CONN_NONE + yield (fd, family, type_, path, raddr, status, pid) + + def retrieve(self, kind, pid=None): + self._procfs_path = get_procfs_path() + if pid is not None: + inodes = self.get_proc_inodes(pid) + if not inodes: + # no connections for this process + return [] + else: + inodes = self.get_all_inodes() + ret = set() + for proto_name, family, type_ in self.tmap[kind]: + path = f"{self._procfs_path}/net/{proto_name}" + if family in {socket.AF_INET, socket.AF_INET6}: + ls = self.process_inet( + path, family, type_, inodes, filter_pid=pid + ) + else: + ls = self.process_unix(path, family, inodes, filter_pid=pid) + for fd, family, type_, laddr, raddr, status, bound_pid in ls: + if pid: + conn = _common.pconn( + fd, family, type_, laddr, raddr, status + ) + else: + conn = _common.sconn( + fd, family, type_, laddr, raddr, status, bound_pid + ) + ret.add(conn) + return list(ret) + + +_net_connections = NetConnections() + + +def net_connections(kind='inet'): + """Return system-wide open connections.""" + return _net_connections.retrieve(kind) + + +def net_io_counters(): + """Return network I/O statistics for every network interface + installed on the system as a dict of raw tuples. + """ + with open_text(f"{get_procfs_path()}/net/dev") as f: + lines = f.readlines() + retdict = {} + for line in lines[2:]: + colon = line.rfind(':') + assert colon > 0, repr(line) + name = line[:colon].strip() + fields = line[colon + 1 :].strip().split() + + ( + # in + bytes_recv, + packets_recv, + errin, + dropin, + _fifoin, # unused + _framein, # unused + _compressedin, # unused + _multicastin, # unused + # out + bytes_sent, + packets_sent, + errout, + dropout, + _fifoout, # unused + _collisionsout, # unused + _carrierout, # unused + _compressedout, # unused + ) = map(int, fields) + + retdict[name] = ( + bytes_sent, + bytes_recv, + packets_sent, + packets_recv, + errin, + errout, + dropin, + dropout, + ) + return retdict + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + duplex_map = { + cext.DUPLEX_FULL: NIC_DUPLEX_FULL, + cext.DUPLEX_HALF: NIC_DUPLEX_HALF, + cext.DUPLEX_UNKNOWN: NIC_DUPLEX_UNKNOWN, + } + names = net_io_counters().keys() + ret = {} + for name in names: + try: + mtu = cext_posix.net_if_mtu(name) + flags = cext_posix.net_if_flags(name) + duplex, speed = cext.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + debug(err) + else: + output_flags = ','.join(flags) + isup = 'running' in flags + ret[name] = _common.snicstats( + isup, duplex_map[duplex], speed, mtu, output_flags + ) + return ret + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_usage = _psposix.disk_usage + + +def disk_io_counters(perdisk=False): + """Return disk I/O statistics for every disk installed on the + system as a dict of raw tuples. + """ + + def read_procfs(): + # OK, this is a bit confusing. The format of /proc/diskstats can + # have 3 variations. + # On Linux 2.4 each line has always 15 fields, e.g.: + # "3 0 8 hda 8 8 8 8 8 8 8 8 8 8 8" + # On Linux 2.6+ each line *usually* has 14 fields, and the disk + # name is in another position, like this: + # "3 0 hda 8 8 8 8 8 8 8 8 8 8 8" + # ...unless (Linux 2.6) the line refers to a partition instead + # of a disk, in which case the line has less fields (7): + # "3 1 hda1 8 8 8 8" + # 4.18+ has 4 fields added: + # "3 0 hda 8 8 8 8 8 8 8 8 8 8 8 0 0 0 0" + # 5.5 has 2 more fields. + # See: + # https://www.kernel.org/doc/Documentation/iostats.txt + # https://www.kernel.org/doc/Documentation/ABI/testing/procfs-diskstats + with open_text(f"{get_procfs_path()}/diskstats") as f: + lines = f.readlines() + for line in lines: + fields = line.split() + flen = len(fields) + # fmt: off + if flen == 15: + # Linux 2.4 + name = fields[3] + reads = int(fields[2]) + (reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields[4:14]) + elif flen == 14 or flen >= 18: + # Linux 2.6+, line referring to a disk + name = fields[2] + (reads, reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time, _) = map(int, fields[3:14]) + elif flen == 7: + # Linux 2.6+, line referring to a partition + name = fields[2] + reads, rbytes, writes, wbytes = map(int, fields[3:]) + rtime = wtime = reads_merged = writes_merged = busy_time = 0 + else: + msg = f"not sure how to interpret line {line!r}" + raise ValueError(msg) + yield (name, reads, writes, rbytes, wbytes, rtime, wtime, + reads_merged, writes_merged, busy_time) + # fmt: on + + def read_sysfs(): + for block in os.listdir('/sys/block'): + for root, _, files in os.walk(os.path.join('/sys/block', block)): + if 'stat' not in files: + continue + with open_text(os.path.join(root, 'stat')) as f: + fields = f.read().strip().split() + name = os.path.basename(root) + # fmt: off + (reads, reads_merged, rbytes, rtime, writes, writes_merged, + wbytes, wtime, _, busy_time) = map(int, fields[:10]) + yield (name, reads, writes, rbytes, wbytes, rtime, + wtime, reads_merged, writes_merged, busy_time) + # fmt: on + + if os.path.exists(f"{get_procfs_path()}/diskstats"): + gen = read_procfs() + elif os.path.exists('/sys/block'): + gen = read_sysfs() + else: + msg = ( + f"{get_procfs_path()}/diskstats nor /sys/block are available on" + " this system" + ) + raise NotImplementedError(msg) + + retdict = {} + for entry in gen: + # fmt: off + (name, reads, writes, rbytes, wbytes, rtime, wtime, reads_merged, + writes_merged, busy_time) = entry + if not perdisk and not is_storage_device(name): + # perdisk=False means we want to calculate totals so we skip + # partitions (e.g. 'sda1', 'nvme0n1p1') and only include + # base disk devices (e.g. 'sda', 'nvme0n1'). Base disks + # include a total of all their partitions + some extra size + # of their own: + # $ cat /proc/diskstats + # 259 0 sda 10485760 ... + # 259 1 sda1 5186039 ... + # 259 1 sda2 5082039 ... + # See: + # https://github.com/giampaolo/psutil/pull/1313 + continue + + rbytes *= DISK_SECTOR_SIZE + wbytes *= DISK_SECTOR_SIZE + retdict[name] = (reads, writes, rbytes, wbytes, rtime, wtime, + reads_merged, writes_merged, busy_time) + # fmt: on + + return retdict + + +class RootFsDeviceFinder: + """disk_partitions() may return partitions with device == "/dev/root" + or "rootfs". This container class uses different strategies to try to + obtain the real device path. Resources: + https://bootlin.com/blog/find-root-device/ + https://www.systutorials.com/how-to-find-the-disk-where-root-is-on-in-bash-on-linux/. + """ + + __slots__ = ['major', 'minor'] + + def __init__(self): + dev = os.stat("/").st_dev + self.major = os.major(dev) + self.minor = os.minor(dev) + + def ask_proc_partitions(self): + with open_text(f"{get_procfs_path()}/partitions") as f: + for line in f.readlines()[2:]: + fields = line.split() + if len(fields) < 4: # just for extra safety + continue + major = int(fields[0]) if fields[0].isdigit() else None + minor = int(fields[1]) if fields[1].isdigit() else None + name = fields[3] + if major == self.major and minor == self.minor: + if name: # just for extra safety + return f"/dev/{name}" + + def ask_sys_dev_block(self): + path = f"/sys/dev/block/{self.major}:{self.minor}/uevent" + with open_text(path) as f: + for line in f: + if line.startswith("DEVNAME="): + name = line.strip().rpartition("DEVNAME=")[2] + if name: # just for extra safety + return f"/dev/{name}" + + def ask_sys_class_block(self): + needle = f"{self.major}:{self.minor}" + files = glob.iglob("/sys/class/block/*/dev") + for file in files: + try: + f = open_text(file) + except FileNotFoundError: # race condition + continue + else: + with f: + data = f.read().strip() + if data == needle: + name = os.path.basename(os.path.dirname(file)) + return f"/dev/{name}" + + def find(self): + path = None + if path is None: + try: + path = self.ask_proc_partitions() + except OSError as err: + debug(err) + if path is None: + try: + path = self.ask_sys_dev_block() + except OSError as err: + debug(err) + if path is None: + try: + path = self.ask_sys_class_block() + except OSError as err: + debug(err) + # We use exists() because the "/dev/*" part of the path is hard + # coded, so we want to be sure. + if path is not None and os.path.exists(path): + return path + + +def disk_partitions(all=False): + """Return mounted disk partitions as a list of namedtuples.""" + fstypes = set() + procfs_path = get_procfs_path() + if not all: + with open_text(f"{procfs_path}/filesystems") as f: + for line in f: + line = line.strip() + if not line.startswith("nodev"): + fstypes.add(line.strip()) + else: + # ignore all lines starting with "nodev" except "nodev zfs" + fstype = line.split("\t")[1] + if fstype == "zfs": + fstypes.add("zfs") + + # See: https://github.com/giampaolo/psutil/issues/1307 + if procfs_path == "/proc" and os.path.isfile('/etc/mtab'): + mounts_path = os.path.realpath("/etc/mtab") + else: + mounts_path = os.path.realpath(f"{procfs_path}/self/mounts") + + retlist = [] + partitions = cext.disk_partitions(mounts_path) + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if device in {"/dev/root", "rootfs"}: + device = RootFsDeviceFinder().find() or device + if not all: + if not device or fstype not in fstypes: + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + + return retlist + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_temperatures(): + """Return hardware (CPU and others) temperatures as a dict + including hardware name, label, current, max and critical + temperatures. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + - /sys/class/thermal/thermal_zone* is another one but it's more + difficult to parse + """ + ret = collections.defaultdict(list) + basenames = glob.glob('/sys/class/hwmon/hwmon*/temp*_*') + # CentOS has an intermediate /device directory: + # https://github.com/giampaolo/psutil/issues/971 + # https://github.com/nicolargo/glances/issues/1060 + basenames.extend(glob.glob('/sys/class/hwmon/hwmon*/device/temp*_*')) + basenames = sorted({x.split('_')[0] for x in basenames}) + + # Only add the coretemp hwmon entries if they're not already in + # /sys/class/hwmon/ + # https://github.com/giampaolo/psutil/issues/1708 + # https://github.com/giampaolo/psutil/pull/1648 + basenames2 = glob.glob( + '/sys/devices/platform/coretemp.*/hwmon/hwmon*/temp*_*' + ) + repl = re.compile(r"/sys/devices/platform/coretemp.*/hwmon/") + for name in basenames2: + altname = repl.sub('/sys/class/hwmon/', name) + if altname not in basenames: + basenames.append(name) + + for base in basenames: + try: + path = base + '_input' + current = float(bcat(path)) / 1000.0 + path = os.path.join(os.path.dirname(base), 'name') + unit_name = cat(path).strip() + except (OSError, ValueError): + # A lot of things can go wrong here, so let's just skip the + # whole entry. Sure thing is Linux's /sys/class/hwmon really + # is a stinky broken mess. + # https://github.com/giampaolo/psutil/issues/1009 + # https://github.com/giampaolo/psutil/issues/1101 + # https://github.com/giampaolo/psutil/issues/1129 + # https://github.com/giampaolo/psutil/issues/1245 + # https://github.com/giampaolo/psutil/issues/1323 + continue + + high = bcat(base + '_max', fallback=None) + critical = bcat(base + '_crit', fallback=None) + label = cat(base + '_label', fallback='').strip() + + if high is not None: + try: + high = float(high) / 1000.0 + except ValueError: + high = None + if critical is not None: + try: + critical = float(critical) / 1000.0 + except ValueError: + critical = None + + ret[unit_name].append((label, current, high, critical)) + + # Indication that no sensors were detected in /sys/class/hwmon/ + if not basenames: + basenames = glob.glob('/sys/class/thermal/thermal_zone*') + basenames = sorted(set(basenames)) + + for base in basenames: + try: + path = os.path.join(base, 'temp') + current = float(bcat(path)) / 1000.0 + path = os.path.join(base, 'type') + unit_name = cat(path).strip() + except (OSError, ValueError) as err: + debug(err) + continue + + trip_paths = glob.glob(base + '/trip_point*') + trip_points = { + '_'.join(os.path.basename(p).split('_')[0:3]) + for p in trip_paths + } + critical = None + high = None + for trip_point in trip_points: + path = os.path.join(base, trip_point + "_type") + trip_type = cat(path, fallback='').strip() + if trip_type == 'critical': + critical = bcat( + os.path.join(base, trip_point + "_temp"), fallback=None + ) + elif trip_type == 'high': + high = bcat( + os.path.join(base, trip_point + "_temp"), fallback=None + ) + + if high is not None: + try: + high = float(high) / 1000.0 + except ValueError: + high = None + if critical is not None: + try: + critical = float(critical) / 1000.0 + except ValueError: + critical = None + + ret[unit_name].append(('', current, high, critical)) + + return dict(ret) + + +def sensors_fans(): + """Return hardware fans info (for CPU and other peripherals) as a + dict including hardware label and current speed. + + Implementation notes: + - /sys/class/hwmon looks like the most recent interface to + retrieve this info, and this implementation relies on it + only (old distros will probably use something else) + - lm-sensors on Ubuntu 16.04 relies on /sys/class/hwmon + """ + ret = collections.defaultdict(list) + basenames = glob.glob('/sys/class/hwmon/hwmon*/fan*_*') + if not basenames: + # CentOS has an intermediate /device directory: + # https://github.com/giampaolo/psutil/issues/971 + basenames = glob.glob('/sys/class/hwmon/hwmon*/device/fan*_*') + + basenames = sorted({x.split("_")[0] for x in basenames}) + for base in basenames: + try: + current = int(bcat(base + '_input')) + except OSError as err: + debug(err) + continue + unit_name = cat(os.path.join(os.path.dirname(base), 'name')).strip() + label = cat(base + '_label', fallback='').strip() + ret[unit_name].append(_common.sfan(label, current)) + + return dict(ret) + + +def sensors_battery(): + """Return battery information. + Implementation note: it appears /sys/class/power_supply/BAT0/ + directory structure may vary and provide files with the same + meaning but under different names, see: + https://github.com/giampaolo/psutil/issues/966. + """ + null = object() + + def multi_bcat(*paths): + """Attempt to read the content of multiple files which may + not exist. If none of them exist return None. + """ + for path in paths: + ret = bcat(path, fallback=null) + if ret != null: + try: + return int(ret) + except ValueError: + return ret.strip() + return None + + bats = [ + x + for x in os.listdir(POWER_SUPPLY_PATH) + if x.startswith('BAT') or 'battery' in x.lower() + ] + if not bats: + return None + # Get the first available battery. Usually this is "BAT0", except + # some rare exceptions: + # https://github.com/giampaolo/psutil/issues/1238 + root = os.path.join(POWER_SUPPLY_PATH, min(bats)) + + # Base metrics. + energy_now = multi_bcat(root + "/energy_now", root + "/charge_now") + power_now = multi_bcat(root + "/power_now", root + "/current_now") + energy_full = multi_bcat(root + "/energy_full", root + "/charge_full") + time_to_empty = multi_bcat(root + "/time_to_empty_now") + + # Percent. If we have energy_full the percentage will be more + # accurate compared to reading /capacity file (float vs. int). + if energy_full is not None and energy_now is not None: + try: + percent = 100.0 * energy_now / energy_full + except ZeroDivisionError: + percent = 0.0 + else: + percent = int(cat(root + "/capacity", fallback=-1)) + if percent == -1: + return None + + # Is AC power cable plugged in? + # Note: AC0 is not always available and sometimes (e.g. CentOS7) + # it's called "AC". + power_plugged = None + online = multi_bcat( + os.path.join(POWER_SUPPLY_PATH, "AC0/online"), + os.path.join(POWER_SUPPLY_PATH, "AC/online"), + ) + if online is not None: + power_plugged = online == 1 + else: + status = cat(root + "/status", fallback="").strip().lower() + if status == "discharging": + power_plugged = False + elif status in {"charging", "full"}: + power_plugged = True + + # Seconds left. + # Note to self: we may also calculate the charging ETA as per: + # https://github.com/thialfihar/dotfiles/blob/ + # 013937745fd9050c30146290e8f963d65c0179e6/bin/battery.py#L55 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + elif energy_now is not None and power_now is not None: + try: + secsleft = int(energy_now / power_now * 3600) + except ZeroDivisionError: + secsleft = _common.POWER_TIME_UNKNOWN + elif time_to_empty is not None: + secsleft = int(time_to_empty * 60) + if secsleft < 0: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = _common.POWER_TIME_UNKNOWN + + return _common.sbattery(percent, secsleft, power_plugged) + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, tty, hostname, tstamp, pid = item + nt = _common.suser(user, tty or None, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +def boot_time(): + """Return the system boot time expressed in seconds since the epoch.""" + global BOOT_TIME + path = f"{get_procfs_path()}/stat" + with open_binary(path) as f: + for line in f: + if line.startswith(b'btime'): + ret = float(line.strip().split()[1]) + BOOT_TIME = ret + return ret + msg = f"line 'btime' not found in {path}" + raise RuntimeError(msg) + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + path = get_procfs_path().encode(ENCODING) + return [int(x) for x in os.listdir(path) if x.isdigit()] + + +def pid_exists(pid): + """Check for the existence of a unix PID. Linux TIDs are not + supported (always return False). + """ + if not _psposix.pid_exists(pid): + return False + else: + # Linux's apparently does not distinguish between PIDs and TIDs + # (thread IDs). + # listdir("/proc") won't show any TID (only PIDs) but + # os.stat("/proc/{tid}") will succeed if {tid} exists. + # os.kill() can also be passed a TID. This is quite confusing. + # In here we want to enforce this distinction and support PIDs + # only, see: + # https://github.com/giampaolo/psutil/issues/687 + try: + # Note: already checked that this is faster than using a + # regular expr. Also (a lot) faster than doing + # 'return pid in pids()' + path = f"{get_procfs_path()}/{pid}/status" + with open_binary(path) as f: + for line in f: + if line.startswith(b"Tgid:"): + tgid = int(line.split()[1]) + # If tgid and pid are the same then we're + # dealing with a process PID. + return tgid == pid + msg = f"'Tgid' line not found in {path}" + raise ValueError(msg) + except (OSError, ValueError): + return pid in pids() + + +def ppid_map(): + """Obtain a {pid: ppid, ...} dict for all running processes in + one shot. Used to speed up Process.children(). + """ + ret = {} + procfs_path = get_procfs_path() + for pid in pids(): + try: + with open_binary(f"{procfs_path}/{pid}/stat") as f: + data = f.read() + except (FileNotFoundError, ProcessLookupError): + # Note: we should be able to access /stat for all processes + # aka it's unlikely we'll bump into EPERM, which is good. + pass + else: + rpar = data.rfind(b')') + dset = data[rpar + 2 :].split() + ppid = int(dset[1]) + ret[pid] = ppid + return ret + + +def wrap_exceptions(fun): + """Decorator which translates bare OSError and OSError exceptions + into NoSuchProcess and AccessDenied. + """ + + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + pid, name = self.pid, self._name + try: + return fun(self, *args, **kwargs) + except PermissionError as err: + raise AccessDenied(pid, name) from err + except ProcessLookupError as err: + self._raise_if_zombie() + raise NoSuchProcess(pid, name) from err + except FileNotFoundError as err: + self._raise_if_zombie() + # /proc/PID directory may still exist, but the files within + # it may not, indicating the process is gone, see: + # https://github.com/giampaolo/psutil/issues/2418 + if not os.path.exists(f"{self._procfs_path}/{pid}/stat"): + raise NoSuchProcess(pid, name) from err + raise + + return wrapper + + +class Process: + """Linux process implementation.""" + + __slots__ = ["_cache", "_name", "_ppid", "_procfs_path", "pid"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + self._procfs_path = get_procfs_path() + + def _is_zombie(self): + # Note: most of the times Linux is able to return info about the + # process even if it's a zombie, and /proc/{pid} will exist. + # There are some exceptions though, like exe(), cmdline() and + # memory_maps(). In these cases /proc/{pid}/{file} exists but + # it's empty. Instead of returning a "null" value we'll raise an + # exception. + try: + data = bcat(f"{self._procfs_path}/{self.pid}/stat") + except OSError: + return False + else: + rpar = data.rfind(b')') + status = data[rpar + 2 : rpar + 3] + return status == b"Z" + + def _raise_if_zombie(self): + if self._is_zombie(): + raise ZombieProcess(self.pid, self._name, self._ppid) + + def _raise_if_not_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + os.stat(f"{self._procfs_path}/{self.pid}") + + @wrap_exceptions + @memoize_when_activated + def _parse_stat_file(self): + """Parse /proc/{pid}/stat file and return a dict with various + process info. + Using "man proc" as a reference: where "man proc" refers to + position N always subtract 3 (e.g ppid position 4 in + 'man proc' == position 1 in here). + The return value is cached in case oneshot() ctx manager is + in use. + """ + data = bcat(f"{self._procfs_path}/{self.pid}/stat") + # Process name is between parentheses. It can contain spaces and + # other parentheses. This is taken into account by looking for + # the first occurrence of "(" and the last occurrence of ")". + rpar = data.rfind(b')') + name = data[data.find(b'(') + 1 : rpar] + fields = data[rpar + 2 :].split() + + ret = {} + ret['name'] = name + ret['status'] = fields[0] + ret['ppid'] = fields[1] + ret['ttynr'] = fields[4] + ret['utime'] = fields[11] + ret['stime'] = fields[12] + ret['children_utime'] = fields[13] + ret['children_stime'] = fields[14] + ret['create_time'] = fields[19] + ret['cpu_num'] = fields[36] + try: + ret['blkio_ticks'] = fields[39] # aka 'delayacct_blkio_ticks' + except IndexError: + # https://github.com/giampaolo/psutil/issues/2455 + debug("can't get blkio_ticks, set iowait to 0") + ret['blkio_ticks'] = 0 + + return ret + + @wrap_exceptions + @memoize_when_activated + def _read_status_file(self): + """Read /proc/{pid}/stat file and return its content. + The return value is cached in case oneshot() ctx manager is + in use. + """ + with open_binary(f"{self._procfs_path}/{self.pid}/status") as f: + return f.read() + + @wrap_exceptions + @memoize_when_activated + def _read_smaps_file(self): + with open_binary(f"{self._procfs_path}/{self.pid}/smaps") as f: + return f.read().strip() + + def oneshot_enter(self): + self._parse_stat_file.cache_activate(self) + self._read_status_file.cache_activate(self) + self._read_smaps_file.cache_activate(self) + + def oneshot_exit(self): + self._parse_stat_file.cache_deactivate(self) + self._read_status_file.cache_deactivate(self) + self._read_smaps_file.cache_deactivate(self) + + @wrap_exceptions + def name(self): + # XXX - gets changed later and probably needs refactoring + return decode(self._parse_stat_file()['name']) + + @wrap_exceptions + def exe(self): + try: + return readlink(f"{self._procfs_path}/{self.pid}/exe") + except (FileNotFoundError, ProcessLookupError): + self._raise_if_zombie() + # no such file error; might be raised also if the + # path actually exists for system processes with + # low pids (about 0-20) + if os.path.lexists(f"{self._procfs_path}/{self.pid}"): + return "" + raise + + @wrap_exceptions + def cmdline(self): + with open_text(f"{self._procfs_path}/{self.pid}/cmdline") as f: + data = f.read() + if not data: + # may happen in case of zombie process + self._raise_if_zombie() + return [] + # 'man proc' states that args are separated by null bytes '\0' + # and last char is supposed to be a null byte. Nevertheless + # some processes may change their cmdline after being started + # (via setproctitle() or similar), they are usually not + # compliant with this rule and use spaces instead. Google + # Chrome process is an example. See: + # https://github.com/giampaolo/psutil/issues/1179 + sep = '\x00' if data.endswith('\x00') else ' ' + if data.endswith(sep): + data = data[:-1] + cmdline = data.split(sep) + # Sometimes last char is a null byte '\0' but the args are + # separated by spaces, see: https://github.com/giampaolo/psutil/ + # issues/1179#issuecomment-552984549 + if sep == '\x00' and len(cmdline) == 1 and ' ' in data: + cmdline = data.split(' ') + return cmdline + + @wrap_exceptions + def environ(self): + with open_text(f"{self._procfs_path}/{self.pid}/environ") as f: + data = f.read() + return parse_environ_block(data) + + @wrap_exceptions + def terminal(self): + tty_nr = int(self._parse_stat_file()['ttynr']) + tmap = _psposix.get_terminal_map() + try: + return tmap[tty_nr] + except KeyError: + return None + + # May not be available on old kernels. + if os.path.exists(f"/proc/{os.getpid()}/io"): + + @wrap_exceptions + def io_counters(self): + fname = f"{self._procfs_path}/{self.pid}/io" + fields = {} + with open_binary(fname) as f: + for line in f: + # https://github.com/giampaolo/psutil/issues/1004 + line = line.strip() + if line: + try: + name, value = line.split(b': ') + except ValueError: + # https://github.com/giampaolo/psutil/issues/1004 + continue + else: + fields[name] = int(value) + if not fields: + msg = f"{fname} file was empty" + raise RuntimeError(msg) + try: + return pio( + fields[b'syscr'], # read syscalls + fields[b'syscw'], # write syscalls + fields[b'read_bytes'], # read bytes + fields[b'write_bytes'], # write bytes + fields[b'rchar'], # read chars + fields[b'wchar'], # write chars + ) + except KeyError as err: + msg = ( + f"{err.args[0]!r} field was not found in {fname}; found" + f" fields are {fields!r}" + ) + raise ValueError(msg) from None + + @wrap_exceptions + def cpu_times(self): + values = self._parse_stat_file() + utime = float(values['utime']) / CLOCK_TICKS + stime = float(values['stime']) / CLOCK_TICKS + children_utime = float(values['children_utime']) / CLOCK_TICKS + children_stime = float(values['children_stime']) / CLOCK_TICKS + iowait = float(values['blkio_ticks']) / CLOCK_TICKS + return pcputimes(utime, stime, children_utime, children_stime, iowait) + + @wrap_exceptions + def cpu_num(self): + """What CPU the process is on.""" + return int(self._parse_stat_file()['cpu_num']) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + @wrap_exceptions + def create_time(self): + ctime = float(self._parse_stat_file()['create_time']) + # According to documentation, starttime is in field 21 and the + # unit is jiffies (clock ticks). + # We first divide it for clock ticks and then add uptime returning + # seconds since the epoch. + # Also use cached value if available. + bt = BOOT_TIME or boot_time() + return (ctime / CLOCK_TICKS) + bt + + @wrap_exceptions + def memory_info(self): + # ============================================================ + # | FIELD | DESCRIPTION | AKA | TOP | + # ============================================================ + # | rss | resident set size | | RES | + # | vms | total program size | size | VIRT | + # | shared | shared pages (from shared mappings) | | SHR | + # | text | text ('code') | trs | CODE | + # | lib | library (unused in Linux 2.6) | lrs | | + # | data | data + stack | drs | DATA | + # | dirty | dirty pages (unused in Linux 2.6) | dt | | + # ============================================================ + with open_binary(f"{self._procfs_path}/{self.pid}/statm") as f: + vms, rss, shared, text, lib, data, dirty = ( + int(x) * PAGESIZE for x in f.readline().split()[:7] + ) + return pmem(rss, vms, shared, text, lib, data, dirty) + + if HAS_PROC_SMAPS_ROLLUP or HAS_PROC_SMAPS: + + def _parse_smaps_rollup(self): + # /proc/pid/smaps_rollup was added to Linux in 2017. Faster + # than /proc/pid/smaps. It reports higher PSS than */smaps + # (from 1k up to 200k higher; tested against all processes). + # IMPORTANT: /proc/pid/smaps_rollup is weird, because it + # raises ESRCH / ENOENT for many PIDs, even if they're alive + # (also as root). In that case we'll use /proc/pid/smaps as + # fallback, which is slower but has a +50% success rate + # compared to /proc/pid/smaps_rollup. + uss = pss = swap = 0 + with open_binary( + f"{self._procfs_path}/{self.pid}/smaps_rollup" + ) as f: + for line in f: + if line.startswith(b"Private_"): + # Private_Clean, Private_Dirty, Private_Hugetlb + uss += int(line.split()[1]) * 1024 + elif line.startswith(b"Pss:"): + pss = int(line.split()[1]) * 1024 + elif line.startswith(b"Swap:"): + swap = int(line.split()[1]) * 1024 + return (uss, pss, swap) + + @wrap_exceptions + def _parse_smaps( + self, + # Gets Private_Clean, Private_Dirty, Private_Hugetlb. + _private_re=re.compile(br"\nPrivate.*:\s+(\d+)"), + _pss_re=re.compile(br"\nPss\:\s+(\d+)"), + _swap_re=re.compile(br"\nSwap\:\s+(\d+)"), + ): + # /proc/pid/smaps does not exist on kernels < 2.6.14 or if + # CONFIG_MMU kernel configuration option is not enabled. + + # Note: using 3 regexes is faster than reading the file + # line by line. + # + # You might be tempted to calculate USS by subtracting + # the "shared" value from the "resident" value in + # /proc//statm. But at least on Linux, statm's "shared" + # value actually counts pages backed by files, which has + # little to do with whether the pages are actually shared. + # /proc/self/smaps on the other hand appears to give us the + # correct information. + smaps_data = self._read_smaps_file() + # Note: smaps file can be empty for certain processes. + # The code below will not crash though and will result to 0. + uss = sum(map(int, _private_re.findall(smaps_data))) * 1024 + pss = sum(map(int, _pss_re.findall(smaps_data))) * 1024 + swap = sum(map(int, _swap_re.findall(smaps_data))) * 1024 + return (uss, pss, swap) + + @wrap_exceptions + def memory_full_info(self): + if HAS_PROC_SMAPS_ROLLUP: # faster + try: + uss, pss, swap = self._parse_smaps_rollup() + except (ProcessLookupError, FileNotFoundError): + uss, pss, swap = self._parse_smaps() + else: + uss, pss, swap = self._parse_smaps() + basic_mem = self.memory_info() + return pfullmem(*basic_mem + (uss, pss, swap)) + + else: + memory_full_info = memory_info + + if HAS_PROC_SMAPS: + + @wrap_exceptions + def memory_maps(self): + """Return process's mapped memory regions as a list of named + tuples. Fields are explained in 'man proc'; here is an updated + (Apr 2012) version: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/tree/Documentation/filesystems/proc.txt?id=b76437579d1344b612cf1851ae610c636cec7db0. + + /proc/{PID}/smaps does not exist on kernels < 2.6.14 or if + CONFIG_MMU kernel configuration option is not enabled. + """ + + def get_blocks(lines, current_block): + data = {} + for line in lines: + fields = line.split(None, 5) + if not fields[0].endswith(b':'): + # new block section + yield (current_block.pop(), data) + current_block.append(line) + else: + try: + data[fields[0]] = int(fields[1]) * 1024 + except ValueError: + if fields[0].startswith(b'VmFlags:'): + # see issue #369 + continue + msg = f"don't know how to interpret line {line!r}" + raise ValueError(msg) from None + yield (current_block.pop(), data) + + data = self._read_smaps_file() + # Note: smaps file can be empty for certain processes or for + # zombies. + if not data: + self._raise_if_zombie() + return [] + lines = data.split(b'\n') + ls = [] + first_line = lines.pop(0) + current_block = [first_line] + for header, data in get_blocks(lines, current_block): + hfields = header.split(None, 5) + try: + addr, perms, _offset, _dev, _inode, path = hfields + except ValueError: + addr, perms, _offset, _dev, _inode, path = hfields + [''] + if not path: + path = '[anon]' + else: + path = decode(path) + path = path.strip() + if path.endswith(' (deleted)') and not path_exists_strict( + path + ): + path = path[:-10] + item = ( + decode(addr), + decode(perms), + path, + data.get(b'Rss:', 0), + data.get(b'Size:', 0), + data.get(b'Pss:', 0), + data.get(b'Shared_Clean:', 0), + data.get(b'Shared_Dirty:', 0), + data.get(b'Private_Clean:', 0), + data.get(b'Private_Dirty:', 0), + data.get(b'Referenced:', 0), + data.get(b'Anonymous:', 0), + data.get(b'Swap:', 0), + ) + ls.append(item) + return ls + + @wrap_exceptions + def cwd(self): + return readlink(f"{self._procfs_path}/{self.pid}/cwd") + + @wrap_exceptions + def num_ctx_switches( + self, _ctxsw_re=re.compile(br'ctxt_switches:\t(\d+)') + ): + data = self._read_status_file() + ctxsw = _ctxsw_re.findall(data) + if not ctxsw: + msg = ( + "'voluntary_ctxt_switches' and" + " 'nonvoluntary_ctxt_switches'lines were not found in" + f" {self._procfs_path}/{self.pid}/status; the kernel is" + " probably older than 2.6.23" + ) + raise NotImplementedError(msg) + return _common.pctxsw(int(ctxsw[0]), int(ctxsw[1])) + + @wrap_exceptions + def num_threads(self, _num_threads_re=re.compile(br'Threads:\t(\d+)')): + # Using a re is faster than iterating over file line by line. + data = self._read_status_file() + return int(_num_threads_re.findall(data)[0]) + + @wrap_exceptions + def threads(self): + thread_ids = os.listdir(f"{self._procfs_path}/{self.pid}/task") + thread_ids.sort() + retlist = [] + hit_enoent = False + for thread_id in thread_ids: + fname = f"{self._procfs_path}/{self.pid}/task/{thread_id}/stat" + try: + with open_binary(fname) as f: + st = f.read().strip() + except (FileNotFoundError, ProcessLookupError): + # no such file or directory or no such process; + # it means thread disappeared on us + hit_enoent = True + continue + # ignore the first two values ("pid (exe)") + st = st[st.find(b')') + 2 :] + values = st.split(b' ') + utime = float(values[11]) / CLOCK_TICKS + stime = float(values[12]) / CLOCK_TICKS + ntuple = _common.pthread(int(thread_id), utime, stime) + retlist.append(ntuple) + if hit_enoent: + self._raise_if_not_alive() + return retlist + + @wrap_exceptions + def nice_get(self): + # with open_text(f"{self._procfs_path}/{self.pid}/stat") as f: + # data = f.read() + # return int(data.split()[18]) + + # Use C implementation + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + # starting from CentOS 6. + if HAS_CPU_AFFINITY: + + @wrap_exceptions + def cpu_affinity_get(self): + return cext.proc_cpu_affinity_get(self.pid) + + def _get_eligible_cpus( + self, _re=re.compile(br"Cpus_allowed_list:\t(\d+)-(\d+)") + ): + # See: https://github.com/giampaolo/psutil/issues/956 + data = self._read_status_file() + match = _re.findall(data) + if match: + return list(range(int(match[0][0]), int(match[0][1]) + 1)) + else: + return list(range(len(per_cpu_times()))) + + @wrap_exceptions + def cpu_affinity_set(self, cpus): + try: + cext.proc_cpu_affinity_set(self.pid, cpus) + except (OSError, ValueError) as err: + if isinstance(err, ValueError) or err.errno == errno.EINVAL: + eligible_cpus = self._get_eligible_cpus() + all_cpus = tuple(range(len(per_cpu_times()))) + for cpu in cpus: + if cpu not in all_cpus: + msg = ( + f"invalid CPU {cpu!r}; choose between" + f" {eligible_cpus!r}" + ) + raise ValueError(msg) from None + if cpu not in eligible_cpus: + msg = ( + f"CPU number {cpu} is not eligible; choose" + f" between {eligible_cpus}" + ) + raise ValueError(msg) from err + raise + + # only starting from kernel 2.6.13 + if HAS_PROC_IO_PRIORITY: + + @wrap_exceptions + def ionice_get(self): + ioclass, value = cext.proc_ioprio_get(self.pid) + ioclass = IOPriority(ioclass) + return _common.pionice(ioclass, value) + + @wrap_exceptions + def ionice_set(self, ioclass, value): + if value is None: + value = 0 + if value and ioclass in { + IOPriority.IOPRIO_CLASS_IDLE, + IOPriority.IOPRIO_CLASS_NONE, + }: + msg = f"{ioclass!r} ioclass accepts no value" + raise ValueError(msg) + if value < 0 or value > 7: + msg = "value not in 0-7 range" + raise ValueError(msg) + return cext.proc_ioprio_set(self.pid, ioclass, value) + + if hasattr(resource, "prlimit"): + + @wrap_exceptions + def rlimit(self, resource_, limits=None): + # If pid is 0 prlimit() applies to the calling process and + # we don't want that. We should never get here though as + # PID 0 is not supported on Linux. + if self.pid == 0: + msg = "can't use prlimit() against PID 0 process" + raise ValueError(msg) + try: + if limits is None: + # get + return resource.prlimit(self.pid, resource_) + else: + # set + if len(limits) != 2: + msg = ( + "second argument must be a (soft, hard) " + f"tuple, got {limits!r}" + ) + raise ValueError(msg) + resource.prlimit(self.pid, resource_, limits) + except OSError as err: + if err.errno == errno.ENOSYS: + # I saw this happening on Travis: + # https://travis-ci.org/giampaolo/psutil/jobs/51368273 + self._raise_if_zombie() + raise + + @wrap_exceptions + def status(self): + letter = self._parse_stat_file()['status'] + letter = letter.decode() + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(letter, '?') + + @wrap_exceptions + def open_files(self): + retlist = [] + files = os.listdir(f"{self._procfs_path}/{self.pid}/fd") + hit_enoent = False + for fd in files: + file = f"{self._procfs_path}/{self.pid}/fd/{fd}" + try: + path = readlink(file) + except (FileNotFoundError, ProcessLookupError): + # ENOENT == file which is gone in the meantime + hit_enoent = True + continue + except OSError as err: + if err.errno == errno.EINVAL: + # not a link + continue + if err.errno == errno.ENAMETOOLONG: + # file name too long + debug(err) + continue + raise + else: + # If path is not an absolute there's no way to tell + # whether it's a regular file or not, so we skip it. + # A regular file is always supposed to be have an + # absolute path though. + if path.startswith('/') and isfile_strict(path): + # Get file position and flags. + file = f"{self._procfs_path}/{self.pid}/fdinfo/{fd}" + try: + with open_binary(file) as f: + pos = int(f.readline().split()[1]) + flags = int(f.readline().split()[1], 8) + except (FileNotFoundError, ProcessLookupError): + # fd gone in the meantime; process may + # still be alive + hit_enoent = True + else: + mode = file_flags_to_mode(flags) + ntuple = popenfile( + path, int(fd), int(pos), mode, flags + ) + retlist.append(ntuple) + if hit_enoent: + self._raise_if_not_alive() + return retlist + + @wrap_exceptions + def net_connections(self, kind='inet'): + ret = _net_connections.retrieve(kind, self.pid) + self._raise_if_not_alive() + return ret + + @wrap_exceptions + def num_fds(self): + return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd")) + + @wrap_exceptions + def ppid(self): + return int(self._parse_stat_file()['ppid']) + + @wrap_exceptions + def uids(self, _uids_re=re.compile(br'Uid:\t(\d+)\t(\d+)\t(\d+)')): + data = self._read_status_file() + real, effective, saved = _uids_re.findall(data)[0] + return _common.puids(int(real), int(effective), int(saved)) + + @wrap_exceptions + def gids(self, _gids_re=re.compile(br'Gid:\t(\d+)\t(\d+)\t(\d+)')): + data = self._read_status_file() + real, effective, saved = _gids_re.findall(data)[0] + return _common.pgids(int(real), int(effective), int(saved)) diff --git a/lib/python3.12/site-packages/psutil/_psosx.py b/lib/python3.12/site-packages/psutil/_psosx.py new file mode 100644 index 0000000000000000000000000000000000000000..620497b30a07bd104c34cc1a3fcb46243b8c9b67 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_psosx.py @@ -0,0 +1,544 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""macOS platform implementation.""" + +import errno +import functools +import os +from collections import namedtuple + +from . import _common +from . import _psposix +from . import _psutil_osx as cext +from . import _psutil_posix as cext_posix +from ._common import AccessDenied +from ._common import NoSuchProcess +from ._common import ZombieProcess +from ._common import conn_tmap +from ._common import conn_to_ntuple +from ._common import isfile_strict +from ._common import memoize_when_activated +from ._common import parse_environ_block +from ._common import usage_percent + + +__extra__all__ = [] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +PAGESIZE = cext_posix.getpagesize() +AF_LINK = cext_posix.AF_LINK + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RECEIVED: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + +PROC_STATUSES = { + cext.SIDL: _common.STATUS_IDLE, + cext.SRUN: _common.STATUS_RUNNING, + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SZOMB: _common.STATUS_ZOMBIE, +} + +kinfo_proc_map = dict( + ppid=0, + ruid=1, + euid=2, + suid=3, + rgid=4, + egid=5, + sgid=6, + ttynr=7, + ctime=8, + status=9, + name=10, +) + +pidtaskinfo_map = dict( + cpuutime=0, + cpustime=1, + rss=2, + vms=3, + pfaults=4, + pageins=5, + numthreads=6, + volctxsw=7, +) + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# fmt: off +# psutil.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'nice', 'system', 'idle']) +# psutil.virtual_memory() +svmem = namedtuple( + 'svmem', ['total', 'available', 'percent', 'used', 'free', + 'active', 'inactive', 'wired']) +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms', 'pfaults', 'pageins']) +# psutil.Process.memory_full_info() +pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', )) +# fmt: on + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + """System virtual memory as a namedtuple.""" + total, active, inactive, wired, free, speculative = cext.virtual_mem() + # This is how Zabbix calculate avail and used mem: + # https://github.com/zabbix/zabbix/blob/master/src/libs/zbxsysinfo/osx/memory.c + # Also see: https://github.com/giampaolo/psutil/issues/1277 + avail = inactive + free + used = active + wired + # This is NOT how Zabbix calculates free mem but it matches "free" + # cmdline utility. + free -= speculative + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, used, free, active, inactive, wired) + + +def swap_memory(): + """Swap system memory as a (total, used, free, sin, sout) tuple.""" + total, used, free, sin, sout = cext.swap_mem() + percent = usage_percent(used, total, round_=1) + return _common.sswap(total, used, free, percent, sin, sout) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system CPU times as a namedtuple.""" + user, nice, system, idle = cext.cpu_times() + return scputimes(user, nice, system, idle) + + +def per_cpu_times(): + """Return system CPU times as a named tuple.""" + ret = [] + for cpu_t in cext.per_cpu_times(): + user, nice, system, idle = cpu_t + item = scputimes(user, nice, system, idle) + ret.append(item) + return ret + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + return cext.cpu_count_logical() + + +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + return cext.cpu_count_cores() + + +def cpu_stats(): + ctx_switches, interrupts, soft_interrupts, syscalls, _traps = ( + cext.cpu_stats() + ) + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls + ) + + +def cpu_freq(): + """Return CPU frequency. + On macOS per-cpu frequency is not supported. + Also, the returned frequency never changes, see: + https://arstechnica.com/civis/viewtopic.php?f=19&t=465002. + """ + curr, min_, max_ = cext.cpu_freq() + return [_common.scpufreq(curr, min_, max_)] + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_usage = _psposix.disk_usage +disk_io_counters = cext.disk_io_counters + + +def disk_partitions(all=False): + """Return mounted disk partitions as a list of namedtuples.""" + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + if not os.path.isabs(device) or not os.path.exists(device): + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + """Return battery information.""" + try: + percent, minsleft, power_plugged = cext.sensors_battery() + except NotImplementedError: + # no power source - return None according to interface + return None + power_plugged = power_plugged == 1 + if power_plugged: + secsleft = _common.POWER_TIME_UNLIMITED + elif minsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + else: + secsleft = minsleft * 60 + return _common.sbattery(percent, secsleft, power_plugged) + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_io_counters = cext.net_io_counters +net_if_addrs = cext_posix.net_if_addrs + + +def net_connections(kind='inet'): + """System-wide network connections.""" + # Note: on macOS this will fail with AccessDenied unless + # the process is owned by root. + ret = [] + for pid in pids(): + try: + cons = Process(pid).net_connections(kind) + except NoSuchProcess: + continue + else: + if cons: + for c in cons: + c = list(c) + [pid] + ret.append(_common.sconn(*c)) + return ret + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + names = net_io_counters().keys() + ret = {} + for name in names: + try: + mtu = cext_posix.net_if_mtu(name) + flags = cext_posix.net_if_flags(name) + duplex, speed = cext_posix.net_if_duplex_speed(name) + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1279 + if err.errno != errno.ENODEV: + raise + else: + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + output_flags = ','.join(flags) + isup = 'running' in flags + ret[name] = _common.snicstats( + isup, duplex, speed, mtu, output_flags + ) + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, tty, hostname, tstamp, pid = item + if tty == '~': + continue # reboot or shutdown + if not tstamp: + continue + nt = _common.suser(user, tty or None, hostname or None, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + ls = cext.pids() + if 0 not in ls: + # On certain macOS versions pids() C doesn't return PID 0 but + # "ps" does and the process is querable via sysctl(): + # https://travis-ci.org/giampaolo/psutil/jobs/309619941 + try: + Process(0).create_time() + ls.insert(0, 0) + except NoSuchProcess: + pass + except AccessDenied: + ls.insert(0, 0) + return ls + + +pid_exists = _psposix.pid_exists + + +def is_zombie(pid): + try: + st = cext.proc_kinfo_oneshot(pid)[kinfo_proc_map['status']] + return st == cext.SZOMB + except OSError: + return False + + +def wrap_exceptions(fun): + """Decorator which translates bare OSError exceptions into + NoSuchProcess and AccessDenied. + """ + + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name + try: + return fun(self, *args, **kwargs) + except ProcessLookupError as err: + if is_zombie(pid): + raise ZombieProcess(pid, name, ppid) from err + raise NoSuchProcess(pid, name) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + + return wrapper + + +class Process: + """Wrapper class around underlying C implementation.""" + + __slots__ = ["_cache", "_name", "_ppid", "pid"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + + @wrap_exceptions + @memoize_when_activated + def _get_kinfo_proc(self): + # Note: should work with all PIDs without permission issues. + ret = cext.proc_kinfo_oneshot(self.pid) + assert len(ret) == len(kinfo_proc_map) + return ret + + @wrap_exceptions + @memoize_when_activated + def _get_pidtaskinfo(self): + # Note: should work for PIDs owned by user only. + ret = cext.proc_pidtaskinfo_oneshot(self.pid) + assert len(ret) == len(pidtaskinfo_map) + return ret + + def oneshot_enter(self): + self._get_kinfo_proc.cache_activate(self) + self._get_pidtaskinfo.cache_activate(self) + + def oneshot_exit(self): + self._get_kinfo_proc.cache_deactivate(self) + self._get_pidtaskinfo.cache_deactivate(self) + + @wrap_exceptions + def name(self): + name = self._get_kinfo_proc()[kinfo_proc_map['name']] + return name if name is not None else cext.proc_name(self.pid) + + @wrap_exceptions + def exe(self): + return cext.proc_exe(self.pid) + + @wrap_exceptions + def cmdline(self): + return cext.proc_cmdline(self.pid) + + @wrap_exceptions + def environ(self): + return parse_environ_block(cext.proc_environ(self.pid)) + + @wrap_exceptions + def ppid(self): + self._ppid = self._get_kinfo_proc()[kinfo_proc_map['ppid']] + return self._ppid + + @wrap_exceptions + def cwd(self): + return cext.proc_cwd(self.pid) + + @wrap_exceptions + def uids(self): + rawtuple = self._get_kinfo_proc() + return _common.puids( + rawtuple[kinfo_proc_map['ruid']], + rawtuple[kinfo_proc_map['euid']], + rawtuple[kinfo_proc_map['suid']], + ) + + @wrap_exceptions + def gids(self): + rawtuple = self._get_kinfo_proc() + return _common.puids( + rawtuple[kinfo_proc_map['rgid']], + rawtuple[kinfo_proc_map['egid']], + rawtuple[kinfo_proc_map['sgid']], + ) + + @wrap_exceptions + def terminal(self): + tty_nr = self._get_kinfo_proc()[kinfo_proc_map['ttynr']] + tmap = _psposix.get_terminal_map() + try: + return tmap[tty_nr] + except KeyError: + return None + + @wrap_exceptions + def memory_info(self): + rawtuple = self._get_pidtaskinfo() + return pmem( + rawtuple[pidtaskinfo_map['rss']], + rawtuple[pidtaskinfo_map['vms']], + rawtuple[pidtaskinfo_map['pfaults']], + rawtuple[pidtaskinfo_map['pageins']], + ) + + @wrap_exceptions + def memory_full_info(self): + basic_mem = self.memory_info() + uss = cext.proc_memory_uss(self.pid) + return pfullmem(*basic_mem + (uss,)) + + @wrap_exceptions + def cpu_times(self): + rawtuple = self._get_pidtaskinfo() + return _common.pcputimes( + rawtuple[pidtaskinfo_map['cpuutime']], + rawtuple[pidtaskinfo_map['cpustime']], + # children user / system times are not retrievable (set to 0) + 0.0, + 0.0, + ) + + @wrap_exceptions + def create_time(self): + return self._get_kinfo_proc()[kinfo_proc_map['ctime']] + + @wrap_exceptions + def num_ctx_switches(self): + # Unvoluntary value seems not to be available; + # getrusage() numbers seems to confirm this theory. + # We set it to 0. + vol = self._get_pidtaskinfo()[pidtaskinfo_map['volctxsw']] + return _common.pctxsw(vol, 0) + + @wrap_exceptions + def num_threads(self): + return self._get_pidtaskinfo()[pidtaskinfo_map['numthreads']] + + @wrap_exceptions + def open_files(self): + if self.pid == 0: + return [] + files = [] + rawlist = cext.proc_open_files(self.pid) + for path, fd in rawlist: + if isfile_strict(path): + ntuple = _common.popenfile(path, fd) + files.append(ntuple) + return files + + @wrap_exceptions + def net_connections(self, kind='inet'): + families, types = conn_tmap[kind] + rawlist = cext.proc_net_connections(self.pid, families, types) + ret = [] + for item in rawlist: + fd, fam, type, laddr, raddr, status = item + nt = conn_to_ntuple( + fd, fam, type, laddr, raddr, status, TCP_STATUSES + ) + ret.append(nt) + return ret + + @wrap_exceptions + def num_fds(self): + if self.pid == 0: + return 0 + return cext.proc_num_fds(self.pid) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) + + @wrap_exceptions + def nice_get(self): + return cext_posix.getpriority(self.pid) + + @wrap_exceptions + def nice_set(self, value): + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def status(self): + code = self._get_kinfo_proc()[kinfo_proc_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + return retlist diff --git a/lib/python3.12/site-packages/psutil/_psposix.py b/lib/python3.12/site-packages/psutil/_psposix.py new file mode 100644 index 0000000000000000000000000000000000000000..88703fdbd26a2b7a117b0d86367ed5dc2d7951aa --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_psposix.py @@ -0,0 +1,207 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Routines common to all posix systems.""" + +import enum +import glob +import os +import signal +import time + +from ._common import MACOS +from ._common import TimeoutExpired +from ._common import memoize +from ._common import sdiskusage +from ._common import usage_percent + + +if MACOS: + from . import _psutil_osx + + +__all__ = ['pid_exists', 'wait_pid', 'disk_usage', 'get_terminal_map'] + + +def pid_exists(pid): + """Check whether pid exists in the current process table.""" + if pid == 0: + # According to "man 2 kill" PID 0 has a special meaning: + # it refers to <> so we don't want to go any further. + # If we get here it means this UNIX platform *does* have + # a process with id 0. + return True + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + # EPERM clearly means there's a process to deny access to + return True + # According to "man 2 kill" possible error values are + # (EINVAL, EPERM, ESRCH) + else: + return True + + +Negsignal = enum.IntEnum( + 'Negsignal', {x.name: -x.value for x in signal.Signals} +) + + +def negsig_to_enum(num): + """Convert a negative signal value to an enum.""" + try: + return Negsignal(num) + except ValueError: + return num + + +def wait_pid( + pid, + timeout=None, + proc_name=None, + _waitpid=os.waitpid, + _timer=getattr(time, 'monotonic', time.time), # noqa: B008 + _min=min, + _sleep=time.sleep, + _pid_exists=pid_exists, +): + """Wait for a process PID to terminate. + + If the process terminated normally by calling exit(3) or _exit(2), + or by returning from main(), the return value is the positive integer + passed to *exit(). + + If it was terminated by a signal it returns the negated value of the + signal which caused the termination (e.g. -SIGTERM). + + If PID is not a children of os.getpid() (current process) just + wait until the process disappears and return None. + + If PID does not exist at all return None immediately. + + If *timeout* != None and process is still alive raise TimeoutExpired. + timeout=0 is also possible (either return immediately or raise). + """ + if pid <= 0: + # see "man waitpid" + msg = "can't wait for PID 0" + raise ValueError(msg) + interval = 0.0001 + flags = 0 + if timeout is not None: + flags |= os.WNOHANG + stop_at = _timer() + timeout + + def sleep(interval): + # Sleep for some time and return a new increased interval. + if timeout is not None: + if _timer() >= stop_at: + raise TimeoutExpired(timeout, pid=pid, name=proc_name) + _sleep(interval) + return _min(interval * 2, 0.04) + + # See: https://linux.die.net/man/2/waitpid + while True: + try: + retpid, status = os.waitpid(pid, flags) + except InterruptedError: + interval = sleep(interval) + except ChildProcessError: + # This has two meanings: + # - PID is not a child of os.getpid() in which case + # we keep polling until it's gone + # - PID never existed in the first place + # In both cases we'll eventually return None as we + # can't determine its exit status code. + while _pid_exists(pid): + interval = sleep(interval) + return None + else: + if retpid == 0: + # WNOHANG flag was used and PID is still running. + interval = sleep(interval) + continue + + if os.WIFEXITED(status): + # Process terminated normally by calling exit(3) or _exit(2), + # or by returning from main(). The return value is the + # positive integer passed to *exit(). + return os.WEXITSTATUS(status) + elif os.WIFSIGNALED(status): + # Process exited due to a signal. Return the negative value + # of that signal. + return negsig_to_enum(-os.WTERMSIG(status)) + # elif os.WIFSTOPPED(status): + # # Process was stopped via SIGSTOP or is being traced, and + # # waitpid() was called with WUNTRACED flag. PID is still + # # alive. From now on waitpid() will keep returning (0, 0) + # # until the process state doesn't change. + # # It may make sense to catch/enable this since stopped PIDs + # # ignore SIGTERM. + # interval = sleep(interval) + # continue + # elif os.WIFCONTINUED(status): + # # Process was resumed via SIGCONT and waitpid() was called + # # with WCONTINUED flag. + # interval = sleep(interval) + # continue + else: + # Should never happen. + msg = f"unknown process exit status {status!r}" + raise ValueError(msg) + + +def disk_usage(path): + """Return disk usage associated with path. + Note: UNIX usually reserves 5% disk space which is not accessible + by user. In this function "total" and "used" values reflect the + total and used disk space whereas "free" and "percent" represent + the "free" and "used percent" user disk space. + """ + st = os.statvfs(path) + # Total space which is only available to root (unless changed + # at system level). + total = st.f_blocks * st.f_frsize + # Remaining free space usable by root. + avail_to_root = st.f_bfree * st.f_frsize + # Remaining free space usable by user. + avail_to_user = st.f_bavail * st.f_frsize + # Total space being used in general. + used = total - avail_to_root + if MACOS: + # see: https://github.com/giampaolo/psutil/pull/2152 + used = _psutil_osx.disk_usage_used(path, used) + # Total space which is available to user (same as 'total' but + # for the user). + total_user = used + avail_to_user + # User usage percent compared to the total amount of space + # the user can use. This number would be higher if compared + # to root's because the user has less space (usually -5%). + usage_percent_user = usage_percent(used, total_user, round_=1) + + # NB: the percentage is -5% than what shown by df due to + # reserved blocks that we are currently not considering: + # https://github.com/giampaolo/psutil/issues/829#issuecomment-223750462 + return sdiskusage( + total=total, used=used, free=avail_to_user, percent=usage_percent_user + ) + + +@memoize +def get_terminal_map(): + """Get a map of device-id -> path as a dict. + Used by Process.terminal(). + """ + ret = {} + ls = glob.glob('/dev/tty*') + glob.glob('/dev/pts/*') + for name in ls: + assert name not in ret, name + try: + ret[os.stat(name).st_rdev] = name + except FileNotFoundError: + pass + return ret diff --git a/lib/python3.12/site-packages/psutil/_pssunos.py b/lib/python3.12/site-packages/psutil/_pssunos.py new file mode 100644 index 0000000000000000000000000000000000000000..78d941cc5ffdc5c08d858f9bd6cfaa4176b8e246 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_pssunos.py @@ -0,0 +1,734 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sun OS Solaris platform implementation.""" + +import errno +import functools +import os +import socket +import subprocess +import sys +from collections import namedtuple +from socket import AF_INET + +from . import _common +from . import _psposix +from . import _psutil_posix as cext_posix +from . import _psutil_sunos as cext +from ._common import AF_INET6 +from ._common import ENCODING +from ._common import AccessDenied +from ._common import NoSuchProcess +from ._common import ZombieProcess +from ._common import debug +from ._common import get_procfs_path +from ._common import isfile_strict +from ._common import memoize_when_activated +from ._common import sockfam_to_enum +from ._common import socktype_to_enum +from ._common import usage_percent + + +__extra__all__ = ["CONN_IDLE", "CONN_BOUND", "PROCFS_PATH"] + + +# ===================================================================== +# --- globals +# ===================================================================== + + +PAGE_SIZE = cext_posix.getpagesize() +AF_LINK = cext_posix.AF_LINK +IS_64_BIT = sys.maxsize > 2**32 + +CONN_IDLE = "IDLE" +CONN_BOUND = "BOUND" + +PROC_STATUSES = { + cext.SSLEEP: _common.STATUS_SLEEPING, + cext.SRUN: _common.STATUS_RUNNING, + cext.SZOMB: _common.STATUS_ZOMBIE, + cext.SSTOP: _common.STATUS_STOPPED, + cext.SIDL: _common.STATUS_IDLE, + cext.SONPROC: _common.STATUS_RUNNING, # same as run + cext.SWAIT: _common.STATUS_WAITING, +} + +TCP_STATUSES = { + cext.TCPS_ESTABLISHED: _common.CONN_ESTABLISHED, + cext.TCPS_SYN_SENT: _common.CONN_SYN_SENT, + cext.TCPS_SYN_RCVD: _common.CONN_SYN_RECV, + cext.TCPS_FIN_WAIT_1: _common.CONN_FIN_WAIT1, + cext.TCPS_FIN_WAIT_2: _common.CONN_FIN_WAIT2, + cext.TCPS_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.TCPS_CLOSED: _common.CONN_CLOSE, + cext.TCPS_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.TCPS_LAST_ACK: _common.CONN_LAST_ACK, + cext.TCPS_LISTEN: _common.CONN_LISTEN, + cext.TCPS_CLOSING: _common.CONN_CLOSING, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, + cext.TCPS_IDLE: CONN_IDLE, # sunos specific + cext.TCPS_BOUND: CONN_BOUND, # sunos specific +} + +proc_info_map = dict( + ppid=0, + rss=1, + vms=2, + create_time=3, + nice=4, + num_threads=5, + status=6, + ttynr=7, + uid=8, + euid=9, + gid=10, + egid=11, +) + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# psutil.cpu_times() +scputimes = namedtuple('scputimes', ['user', 'system', 'idle', 'iowait']) +# psutil.cpu_times(percpu=True) +pcputimes = namedtuple( + 'pcputimes', ['user', 'system', 'children_user', 'children_system'] +) +# psutil.virtual_memory() +svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) +# psutil.Process.memory_info() +pmem = namedtuple('pmem', ['rss', 'vms']) +pfullmem = pmem +# psutil.Process.memory_maps(grouped=True) +pmmap_grouped = namedtuple( + 'pmmap_grouped', ['path', 'rss', 'anonymous', 'locked'] +) +# psutil.Process.memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields) +) + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + """Report virtual memory metrics.""" + # we could have done this with kstat, but IMHO this is good enough + total = os.sysconf('SC_PHYS_PAGES') * PAGE_SIZE + # note: there's no difference on Solaris + free = avail = os.sysconf('SC_AVPHYS_PAGES') * PAGE_SIZE + used = total - free + percent = usage_percent(used, total, round_=1) + return svmem(total, avail, percent, used, free) + + +def swap_memory(): + """Report swap memory metrics.""" + sin, sout = cext.swap_mem() + # XXX + # we are supposed to get total/free by doing so: + # http://cvs.opensolaris.org/source/xref/onnv/onnv-gate/ + # usr/src/cmd/swap/swap.c + # ...nevertheless I can't manage to obtain the same numbers as 'swap' + # cmdline utility, so let's parse its output (sigh!) + p = subprocess.Popen( + [ + '/usr/bin/env', + f"PATH=/usr/sbin:/sbin:{os.environ['PATH']}", + 'swap', + '-l', + ], + stdout=subprocess.PIPE, + ) + stdout, _ = p.communicate() + stdout = stdout.decode(sys.stdout.encoding) + if p.returncode != 0: + msg = f"'swap -l' failed (retcode={p.returncode})" + raise RuntimeError(msg) + + lines = stdout.strip().split('\n')[1:] + if not lines: + msg = 'no swap device(s) configured' + raise RuntimeError(msg) + total = free = 0 + for line in lines: + line = line.split() + t, f = line[3:5] + total += int(int(t) * 512) + free += int(int(f) * 512) + used = total - free + percent = usage_percent(used, total, round_=1) + return _common.sswap( + total, used, free, percent, sin * PAGE_SIZE, sout * PAGE_SIZE + ) + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system-wide CPU times as a named tuple.""" + ret = cext.per_cpu_times() + return scputimes(*[sum(x) for x in zip(*ret)]) + + +def per_cpu_times(): + """Return system per-CPU times as a list of named tuples.""" + ret = cext.per_cpu_times() + return [scputimes(*x) for x in ret] + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + try: + return os.sysconf("SC_NPROCESSORS_ONLN") + except ValueError: + # mimic os.cpu_count() behavior + return None + + +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + return cext.cpu_count_cores() + + +def cpu_stats(): + """Return various CPU stats as a named tuple.""" + ctx_switches, interrupts, syscalls, _traps = cext.cpu_stats() + soft_interrupts = 0 + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls + ) + + +# ===================================================================== +# --- disks +# ===================================================================== + + +disk_io_counters = cext.disk_io_counters +disk_usage = _psposix.disk_usage + + +def disk_partitions(all=False): + """Return system disk partitions.""" + # TODO - the filtering logic should be better checked so that + # it tries to reflect 'df' as much as possible + retlist = [] + partitions = cext.disk_partitions() + for partition in partitions: + device, mountpoint, fstype, opts = partition + if device == 'none': + device = '' + if not all: + # Differently from, say, Linux, we don't have a list of + # common fs types so the best we can do, AFAIK, is to + # filter by filesystem having a total size > 0. + try: + if not disk_usage(mountpoint).total: + continue + except OSError as err: + # https://github.com/giampaolo/psutil/issues/1674 + debug(f"skipping {mountpoint!r}: {err}") + continue + ntuple = _common.sdiskpart(device, mountpoint, fstype, opts) + retlist.append(ntuple) + return retlist + + +# ===================================================================== +# --- network +# ===================================================================== + + +net_io_counters = cext.net_io_counters +net_if_addrs = cext_posix.net_if_addrs + + +def net_connections(kind, _pid=-1): + """Return socket connections. If pid == -1 return system-wide + connections (as opposed to connections opened by one process only). + Only INET sockets are returned (UNIX are not). + """ + families, types = _common.conn_tmap[kind] + rawlist = cext.net_connections(_pid) + ret = set() + for item in rawlist: + fd, fam, type_, laddr, raddr, status, pid = item + if fam not in families: + continue + if type_ not in types: + continue + # TODO: refactor and use _common.conn_to_ntuple. + if fam in {AF_INET, AF_INET6}: + if laddr: + laddr = _common.addr(*laddr) + if raddr: + raddr = _common.addr(*raddr) + status = TCP_STATUSES[status] + fam = sockfam_to_enum(fam) + type_ = socktype_to_enum(type_) + if _pid == -1: + nt = _common.sconn(fd, fam, type_, laddr, raddr, status, pid) + else: + nt = _common.pconn(fd, fam, type_, laddr, raddr, status) + ret.add(nt) + return list(ret) + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + ret = cext.net_if_stats() + for name, items in ret.items(): + isup, duplex, speed, mtu = items + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu, '') + return ret + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + return cext.boot_time() + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + localhost = (':0.0', ':0') + for item in rawlist: + user, tty, hostname, tstamp, user_process, pid = item + # note: the underlying C function includes entries about + # system boot, run level and others. We might want + # to use them in the future. + if not user_process: + continue + if hostname in localhost: + hostname = 'localhost' + nt = _common.suser(user, tty, hostname, tstamp, pid) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- processes +# ===================================================================== + + +def pids(): + """Returns a list of PIDs currently running on the system.""" + path = get_procfs_path().encode(ENCODING) + return [int(x) for x in os.listdir(path) if x.isdigit()] + + +def pid_exists(pid): + """Check for the existence of a unix pid.""" + return _psposix.pid_exists(pid) + + +def wrap_exceptions(fun): + """Call callable into a try/except clause and translate ENOENT, + EACCES and EPERM in NoSuchProcess or AccessDenied exceptions. + """ + + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + pid, ppid, name = self.pid, self._ppid, self._name + try: + return fun(self, *args, **kwargs) + except (FileNotFoundError, ProcessLookupError) as err: + # ENOENT (no such file or directory) gets raised on open(). + # ESRCH (no such process) can get raised on read() if + # process is gone in meantime. + if not pid_exists(pid): + raise NoSuchProcess(pid, name) from err + raise ZombieProcess(pid, name, ppid) from err + except PermissionError as err: + raise AccessDenied(pid, name) from err + except OSError as err: + if pid == 0: + if 0 in pids(): + raise AccessDenied(pid, name) from err + raise + raise + + return wrapper + + +class Process: + """Wrapper class around underlying C implementation.""" + + __slots__ = ["_cache", "_name", "_ppid", "_procfs_path", "pid"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + self._procfs_path = get_procfs_path() + + def _assert_alive(self): + """Raise NSP if the process disappeared on us.""" + # For those C function who do not raise NSP, possibly returning + # incorrect or incomplete result. + os.stat(f"{self._procfs_path}/{self.pid}") + + def oneshot_enter(self): + self._proc_name_and_args.cache_activate(self) + self._proc_basic_info.cache_activate(self) + self._proc_cred.cache_activate(self) + + def oneshot_exit(self): + self._proc_name_and_args.cache_deactivate(self) + self._proc_basic_info.cache_deactivate(self) + self._proc_cred.cache_deactivate(self) + + @wrap_exceptions + @memoize_when_activated + def _proc_name_and_args(self): + return cext.proc_name_and_args(self.pid, self._procfs_path) + + @wrap_exceptions + @memoize_when_activated + def _proc_basic_info(self): + if self.pid == 0 and not os.path.exists( + f"{self._procfs_path}/{self.pid}/psinfo" + ): + raise AccessDenied(self.pid) + ret = cext.proc_basic_info(self.pid, self._procfs_path) + assert len(ret) == len(proc_info_map) + return ret + + @wrap_exceptions + @memoize_when_activated + def _proc_cred(self): + return cext.proc_cred(self.pid, self._procfs_path) + + @wrap_exceptions + def name(self): + # note: max len == 15 + return self._proc_name_and_args()[0] + + @wrap_exceptions + def exe(self): + try: + return os.readlink(f"{self._procfs_path}/{self.pid}/path/a.out") + except OSError: + pass # continue and guess the exe name from the cmdline + # Will be guessed later from cmdline but we want to explicitly + # invoke cmdline here in order to get an AccessDenied + # exception if the user has not enough privileges. + self.cmdline() + return "" + + @wrap_exceptions + def cmdline(self): + return self._proc_name_and_args()[1].split(' ') + + @wrap_exceptions + def environ(self): + return cext.proc_environ(self.pid, self._procfs_path) + + @wrap_exceptions + def create_time(self): + return self._proc_basic_info()[proc_info_map['create_time']] + + @wrap_exceptions + def num_threads(self): + return self._proc_basic_info()[proc_info_map['num_threads']] + + @wrap_exceptions + def nice_get(self): + # Note #1: getpriority(3) doesn't work for realtime processes. + # Psinfo is what ps uses, see: + # https://github.com/giampaolo/psutil/issues/1194 + return self._proc_basic_info()[proc_info_map['nice']] + + @wrap_exceptions + def nice_set(self, value): + if self.pid in {2, 3}: + # Special case PIDs: internally setpriority(3) return ESRCH + # (no such process), no matter what. + # The process actually exists though, as it has a name, + # creation time, etc. + raise AccessDenied(self.pid, self._name) + return cext_posix.setpriority(self.pid, value) + + @wrap_exceptions + def ppid(self): + self._ppid = self._proc_basic_info()[proc_info_map['ppid']] + return self._ppid + + @wrap_exceptions + def uids(self): + try: + real, effective, saved, _, _, _ = self._proc_cred() + except AccessDenied: + real = self._proc_basic_info()[proc_info_map['uid']] + effective = self._proc_basic_info()[proc_info_map['euid']] + saved = None + return _common.puids(real, effective, saved) + + @wrap_exceptions + def gids(self): + try: + _, _, _, real, effective, saved = self._proc_cred() + except AccessDenied: + real = self._proc_basic_info()[proc_info_map['gid']] + effective = self._proc_basic_info()[proc_info_map['egid']] + saved = None + return _common.puids(real, effective, saved) + + @wrap_exceptions + def cpu_times(self): + try: + times = cext.proc_cpu_times(self.pid, self._procfs_path) + except OSError as err: + if err.errno == errno.EOVERFLOW and not IS_64_BIT: + # We may get here if we attempt to query a 64bit process + # with a 32bit python. + # Error originates from read() and also tools like "cat" + # fail in the same way (!). + # Since there simply is no way to determine CPU times we + # return 0.0 as a fallback. See: + # https://github.com/giampaolo/psutil/issues/857 + times = (0.0, 0.0, 0.0, 0.0) + else: + raise + return _common.pcputimes(*times) + + @wrap_exceptions + def cpu_num(self): + return cext.proc_cpu_num(self.pid, self._procfs_path) + + @wrap_exceptions + def terminal(self): + procfs_path = self._procfs_path + hit_enoent = False + tty = wrap_exceptions(self._proc_basic_info()[proc_info_map['ttynr']]) + if tty != cext.PRNODEV: + for x in (0, 1, 2, 255): + try: + return os.readlink(f"{procfs_path}/{self.pid}/path/{x}") + except FileNotFoundError: + hit_enoent = True + continue + if hit_enoent: + self._assert_alive() + + @wrap_exceptions + def cwd(self): + # /proc/PID/path/cwd may not be resolved by readlink() even if + # it exists (ls shows it). If that's the case and the process + # is still alive return None (we can return None also on BSD). + # Reference: https://groups.google.com/g/comp.unix.solaris/c/tcqvhTNFCAs + procfs_path = self._procfs_path + try: + return os.readlink(f"{procfs_path}/{self.pid}/path/cwd") + except FileNotFoundError: + os.stat(f"{procfs_path}/{self.pid}") # raise NSP or AD + return "" + + @wrap_exceptions + def memory_info(self): + ret = self._proc_basic_info() + rss = ret[proc_info_map['rss']] * 1024 + vms = ret[proc_info_map['vms']] * 1024 + return pmem(rss, vms) + + memory_full_info = memory_info + + @wrap_exceptions + def status(self): + code = self._proc_basic_info()[proc_info_map['status']] + # XXX is '?' legit? (we're not supposed to return it anyway) + return PROC_STATUSES.get(code, '?') + + @wrap_exceptions + def threads(self): + procfs_path = self._procfs_path + ret = [] + tids = os.listdir(f"{procfs_path}/{self.pid}/lwp") + hit_enoent = False + for tid in tids: + tid = int(tid) + try: + utime, stime = cext.query_process_thread( + self.pid, tid, procfs_path + ) + except OSError as err: + if err.errno == errno.EOVERFLOW and not IS_64_BIT: + # We may get here if we attempt to query a 64bit process + # with a 32bit python. + # Error originates from read() and also tools like "cat" + # fail in the same way (!). + # Since there simply is no way to determine CPU times we + # return 0.0 as a fallback. See: + # https://github.com/giampaolo/psutil/issues/857 + continue + # ENOENT == thread gone in meantime + if err.errno == errno.ENOENT: + hit_enoent = True + continue + raise + else: + nt = _common.pthread(tid, utime, stime) + ret.append(nt) + if hit_enoent: + self._assert_alive() + return ret + + @wrap_exceptions + def open_files(self): + retlist = [] + hit_enoent = False + procfs_path = self._procfs_path + pathdir = f"{procfs_path}/{self.pid}/path" + for fd in os.listdir(f"{procfs_path}/{self.pid}/fd"): + path = os.path.join(pathdir, fd) + if os.path.islink(path): + try: + file = os.readlink(path) + except FileNotFoundError: + hit_enoent = True + continue + else: + if isfile_strict(file): + retlist.append(_common.popenfile(file, int(fd))) + if hit_enoent: + self._assert_alive() + return retlist + + def _get_unix_sockets(self, pid): + """Get UNIX sockets used by process by parsing 'pfiles' output.""" + # TODO: rewrite this in C (...but the damn netstat source code + # does not include this part! Argh!!) + cmd = ["pfiles", str(pid)] + p = subprocess.Popen( + cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + stdout, stderr = p.communicate() + stdout, stderr = ( + x.decode(sys.stdout.encoding) for x in (stdout, stderr) + ) + if p.returncode != 0: + if 'permission denied' in stderr.lower(): + raise AccessDenied(self.pid, self._name) + if 'no such process' in stderr.lower(): + raise NoSuchProcess(self.pid, self._name) + msg = f"{cmd!r} command error\n{stderr}" + raise RuntimeError(msg) + + lines = stdout.split('\n')[2:] + for i, line in enumerate(lines): + line = line.lstrip() + if line.startswith('sockname: AF_UNIX'): + path = line.split(' ', 2)[2] + type = lines[i - 2].strip() + if type == 'SOCK_STREAM': + type = socket.SOCK_STREAM + elif type == 'SOCK_DGRAM': + type = socket.SOCK_DGRAM + else: + type = -1 + yield (-1, socket.AF_UNIX, type, path, "", _common.CONN_NONE) + + @wrap_exceptions + def net_connections(self, kind='inet'): + ret = net_connections(kind, _pid=self.pid) + # The underlying C implementation retrieves all OS connections + # and filters them by PID. At this point we can't tell whether + # an empty list means there were no connections for process or + # process is no longer active so we force NSP in case the PID + # is no longer there. + if not ret: + # will raise NSP if process is gone + os.stat(f"{self._procfs_path}/{self.pid}") + + # UNIX sockets + if kind in {'all', 'unix'}: + ret.extend([ + _common.pconn(*conn) + for conn in self._get_unix_sockets(self.pid) + ]) + return ret + + nt_mmap_grouped = namedtuple('mmap', 'path rss anon locked') + nt_mmap_ext = namedtuple('mmap', 'addr perms path rss anon locked') + + @wrap_exceptions + def memory_maps(self): + def toaddr(start, end): + return "{}-{}".format( + hex(start)[2:].strip('L'), hex(end)[2:].strip('L') + ) + + procfs_path = self._procfs_path + retlist = [] + try: + rawlist = cext.proc_memory_maps(self.pid, procfs_path) + except OSError as err: + if err.errno == errno.EOVERFLOW and not IS_64_BIT: + # We may get here if we attempt to query a 64bit process + # with a 32bit python. + # Error originates from read() and also tools like "cat" + # fail in the same way (!). + # Since there simply is no way to determine CPU times we + # return 0.0 as a fallback. See: + # https://github.com/giampaolo/psutil/issues/857 + return [] + else: + raise + hit_enoent = False + for item in rawlist: + addr, addrsize, perm, name, rss, anon, locked = item + addr = toaddr(addr, addrsize) + if not name.startswith('['): + try: + name = os.readlink(f"{procfs_path}/{self.pid}/path/{name}") + except OSError as err: + if err.errno == errno.ENOENT: + # sometimes the link may not be resolved by + # readlink() even if it exists (ls shows it). + # If that's the case we just return the + # unresolved link path. + # This seems an inconsistency with /proc similar + # to: http://goo.gl/55XgO + name = f"{procfs_path}/{self.pid}/path/{name}" + hit_enoent = True + else: + raise + retlist.append((addr, perm, name, rss, anon, locked)) + if hit_enoent: + self._assert_alive() + return retlist + + @wrap_exceptions + def num_fds(self): + return len(os.listdir(f"{self._procfs_path}/{self.pid}/fd")) + + @wrap_exceptions + def num_ctx_switches(self): + return _common.pctxsw( + *cext.proc_num_ctx_switches(self.pid, self._procfs_path) + ) + + @wrap_exceptions + def wait(self, timeout=None): + return _psposix.wait_pid(self.pid, timeout, self._name) diff --git a/lib/python3.12/site-packages/psutil/_psutil_posix.abi3.so b/lib/python3.12/site-packages/psutil/_psutil_posix.abi3.so new file mode 100644 index 0000000000000000000000000000000000000000..70bd2966a422d59e8750d4c1d12e833add3d065b Binary files /dev/null and b/lib/python3.12/site-packages/psutil/_psutil_posix.abi3.so differ diff --git a/lib/python3.12/site-packages/psutil/_pswindows.py b/lib/python3.12/site-packages/psutil/_pswindows.py new file mode 100644 index 0000000000000000000000000000000000000000..e5af3c90f4311077d0134575f6e7ff25d89baee5 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/_pswindows.py @@ -0,0 +1,1103 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Windows platform implementation.""" + +import contextlib +import enum +import functools +import os +import signal +import sys +import time +from collections import namedtuple + +from . import _common +from ._common import ENCODING +from ._common import AccessDenied +from ._common import NoSuchProcess +from ._common import TimeoutExpired +from ._common import conn_tmap +from ._common import conn_to_ntuple +from ._common import debug +from ._common import isfile_strict +from ._common import memoize +from ._common import memoize_when_activated +from ._common import parse_environ_block +from ._common import usage_percent +from ._psutil_windows import ABOVE_NORMAL_PRIORITY_CLASS +from ._psutil_windows import BELOW_NORMAL_PRIORITY_CLASS +from ._psutil_windows import HIGH_PRIORITY_CLASS +from ._psutil_windows import IDLE_PRIORITY_CLASS +from ._psutil_windows import NORMAL_PRIORITY_CLASS +from ._psutil_windows import REALTIME_PRIORITY_CLASS + + +try: + from . import _psutil_windows as cext +except ImportError as err: + if ( + str(err).lower().startswith("dll load failed") + and sys.getwindowsversion()[0] < 6 + ): + # We may get here if: + # 1) we are on an old Windows version + # 2) psutil was installed via pip + wheel + # See: https://github.com/giampaolo/psutil/issues/811 + msg = "this Windows version is too old (< Windows Vista); " + msg += "psutil 3.4.2 is the latest version which supports Windows " + msg += "2000, XP and 2003 server" + raise RuntimeError(msg) from err + else: + raise + + +# process priority constants, import from __init__.py: +# http://msdn.microsoft.com/en-us/library/ms686219(v=vs.85).aspx +# fmt: off +__extra__all__ = [ + "win_service_iter", "win_service_get", + # Process priority + "ABOVE_NORMAL_PRIORITY_CLASS", "BELOW_NORMAL_PRIORITY_CLASS", + "HIGH_PRIORITY_CLASS", "IDLE_PRIORITY_CLASS", "NORMAL_PRIORITY_CLASS", + "REALTIME_PRIORITY_CLASS", + # IO priority + "IOPRIO_VERYLOW", "IOPRIO_LOW", "IOPRIO_NORMAL", "IOPRIO_HIGH", + # others + "CONN_DELETE_TCB", "AF_LINK", +] +# fmt: on + + +# ===================================================================== +# --- globals +# ===================================================================== + +CONN_DELETE_TCB = "DELETE_TCB" +ERROR_PARTIAL_COPY = 299 +PYPY = '__pypy__' in sys.builtin_module_names + +AddressFamily = enum.IntEnum('AddressFamily', {'AF_LINK': -1}) +AF_LINK = AddressFamily.AF_LINK + +TCP_STATUSES = { + cext.MIB_TCP_STATE_ESTAB: _common.CONN_ESTABLISHED, + cext.MIB_TCP_STATE_SYN_SENT: _common.CONN_SYN_SENT, + cext.MIB_TCP_STATE_SYN_RCVD: _common.CONN_SYN_RECV, + cext.MIB_TCP_STATE_FIN_WAIT1: _common.CONN_FIN_WAIT1, + cext.MIB_TCP_STATE_FIN_WAIT2: _common.CONN_FIN_WAIT2, + cext.MIB_TCP_STATE_TIME_WAIT: _common.CONN_TIME_WAIT, + cext.MIB_TCP_STATE_CLOSED: _common.CONN_CLOSE, + cext.MIB_TCP_STATE_CLOSE_WAIT: _common.CONN_CLOSE_WAIT, + cext.MIB_TCP_STATE_LAST_ACK: _common.CONN_LAST_ACK, + cext.MIB_TCP_STATE_LISTEN: _common.CONN_LISTEN, + cext.MIB_TCP_STATE_CLOSING: _common.CONN_CLOSING, + cext.MIB_TCP_STATE_DELETE_TCB: CONN_DELETE_TCB, + cext.PSUTIL_CONN_NONE: _common.CONN_NONE, +} + + +class Priority(enum.IntEnum): + ABOVE_NORMAL_PRIORITY_CLASS = ABOVE_NORMAL_PRIORITY_CLASS + BELOW_NORMAL_PRIORITY_CLASS = BELOW_NORMAL_PRIORITY_CLASS + HIGH_PRIORITY_CLASS = HIGH_PRIORITY_CLASS + IDLE_PRIORITY_CLASS = IDLE_PRIORITY_CLASS + NORMAL_PRIORITY_CLASS = NORMAL_PRIORITY_CLASS + REALTIME_PRIORITY_CLASS = REALTIME_PRIORITY_CLASS + + +globals().update(Priority.__members__) + + +class IOPriority(enum.IntEnum): + IOPRIO_VERYLOW = 0 + IOPRIO_LOW = 1 + IOPRIO_NORMAL = 2 + IOPRIO_HIGH = 3 + + +globals().update(IOPriority.__members__) + +pinfo_map = dict( + num_handles=0, + ctx_switches=1, + user_time=2, + kernel_time=3, + create_time=4, + num_threads=5, + io_rcount=6, + io_wcount=7, + io_rbytes=8, + io_wbytes=9, + io_count_others=10, + io_bytes_others=11, + num_page_faults=12, + peak_wset=13, + wset=14, + peak_paged_pool=15, + paged_pool=16, + peak_non_paged_pool=17, + non_paged_pool=18, + pagefile=19, + peak_pagefile=20, + mem_private=21, +) + + +# ===================================================================== +# --- named tuples +# ===================================================================== + + +# fmt: off +# psutil.cpu_times() +scputimes = namedtuple('scputimes', + ['user', 'system', 'idle', 'interrupt', 'dpc']) +# psutil.virtual_memory() +svmem = namedtuple('svmem', ['total', 'available', 'percent', 'used', 'free']) +# psutil.Process.memory_info() +pmem = namedtuple( + 'pmem', ['rss', 'vms', + 'num_page_faults', 'peak_wset', 'wset', 'peak_paged_pool', + 'paged_pool', 'peak_nonpaged_pool', 'nonpaged_pool', + 'pagefile', 'peak_pagefile', 'private']) +# psutil.Process.memory_full_info() +pfullmem = namedtuple('pfullmem', pmem._fields + ('uss', )) +# psutil.Process.memory_maps(grouped=True) +pmmap_grouped = namedtuple('pmmap_grouped', ['path', 'rss']) +# psutil.Process.memory_maps(grouped=False) +pmmap_ext = namedtuple( + 'pmmap_ext', 'addr perms ' + ' '.join(pmmap_grouped._fields)) +# psutil.Process.io_counters() +pio = namedtuple('pio', ['read_count', 'write_count', + 'read_bytes', 'write_bytes', + 'other_count', 'other_bytes']) +# fmt: on + + +# ===================================================================== +# --- utils +# ===================================================================== + + +@functools.lru_cache(maxsize=512) +def convert_dos_path(s): + r"""Convert paths using native DOS format like: + "\Device\HarddiskVolume1\Windows\systemew\file.txt" + into: + "C:\Windows\systemew\file.txt". + """ + rawdrive = '\\'.join(s.split('\\')[:3]) + driveletter = cext.QueryDosDevice(rawdrive) + remainder = s[len(rawdrive) :] + return os.path.join(driveletter, remainder) + + +@memoize +def getpagesize(): + return cext.getpagesize() + + +# ===================================================================== +# --- memory +# ===================================================================== + + +def virtual_memory(): + """System virtual memory as a namedtuple.""" + mem = cext.virtual_mem() + totphys, availphys, _totsys, _availsys = mem + total = totphys + avail = availphys + free = availphys + used = total - avail + percent = usage_percent((total - avail), total, round_=1) + return svmem(total, avail, percent, used, free) + + +def swap_memory(): + """Swap system memory as a (total, used, free, sin, sout) tuple.""" + mem = cext.virtual_mem() + + total_phys = mem[0] + total_system = mem[2] + + # system memory (commit total/limit) is the sum of physical and swap + # thus physical memory values need to be subtracted to get swap values + total = total_system - total_phys + # commit total is incremented immediately (decrementing free_system) + # while the corresponding free physical value is not decremented until + # pages are accessed, so we can't use free system memory for swap. + # instead, we calculate page file usage based on performance counter + if total > 0: + percentswap = cext.swap_percent() + used = int(0.01 * percentswap * total) + else: + percentswap = 0.0 + used = 0 + + free = total - used + percent = round(percentswap, 1) + return _common.sswap(total, used, free, percent, 0, 0) + + +# ===================================================================== +# --- disk +# ===================================================================== + + +disk_io_counters = cext.disk_io_counters + + +def disk_usage(path): + """Return disk usage associated with path.""" + if isinstance(path, bytes): + # XXX: do we want to use "strict"? Probably yes, in order + # to fail immediately. After all we are accepting input here... + path = path.decode(ENCODING, errors="strict") + total, free = cext.disk_usage(path) + used = total - free + percent = usage_percent(used, total, round_=1) + return _common.sdiskusage(total, used, free, percent) + + +def disk_partitions(all): + """Return disk partitions.""" + rawlist = cext.disk_partitions(all) + return [_common.sdiskpart(*x) for x in rawlist] + + +# ===================================================================== +# --- CPU +# ===================================================================== + + +def cpu_times(): + """Return system CPU times as a named tuple.""" + user, system, idle = cext.cpu_times() + # Internally, GetSystemTimes() is used, and it doesn't return + # interrupt and dpc times. cext.per_cpu_times() does, so we + # rely on it to get those only. + percpu_summed = scputimes(*[sum(n) for n in zip(*cext.per_cpu_times())]) + return scputimes( + user, system, idle, percpu_summed.interrupt, percpu_summed.dpc + ) + + +def per_cpu_times(): + """Return system per-CPU times as a list of named tuples.""" + ret = [] + for user, system, idle, interrupt, dpc in cext.per_cpu_times(): + item = scputimes(user, system, idle, interrupt, dpc) + ret.append(item) + return ret + + +def cpu_count_logical(): + """Return the number of logical CPUs in the system.""" + return cext.cpu_count_logical() + + +def cpu_count_cores(): + """Return the number of CPU cores in the system.""" + return cext.cpu_count_cores() + + +def cpu_stats(): + """Return CPU statistics.""" + ctx_switches, interrupts, _dpcs, syscalls = cext.cpu_stats() + soft_interrupts = 0 + return _common.scpustats( + ctx_switches, interrupts, soft_interrupts, syscalls + ) + + +def cpu_freq(): + """Return CPU frequency. + On Windows per-cpu frequency is not supported. + """ + curr, max_ = cext.cpu_freq() + min_ = 0.0 + return [_common.scpufreq(float(curr), min_, float(max_))] + + +_loadavg_inititialized = False + + +def getloadavg(): + """Return the number of processes in the system run queue averaged + over the last 1, 5, and 15 minutes respectively as a tuple. + """ + global _loadavg_inititialized + + if not _loadavg_inititialized: + cext.init_loadavg_counter() + _loadavg_inititialized = True + + # Drop to 2 decimal points which is what Linux does + raw_loads = cext.getloadavg() + return tuple(round(load, 2) for load in raw_loads) + + +# ===================================================================== +# --- network +# ===================================================================== + + +def net_connections(kind, _pid=-1): + """Return socket connections. If pid == -1 return system-wide + connections (as opposed to connections opened by one process only). + """ + families, types = conn_tmap[kind] + rawlist = cext.net_connections(_pid, families, types) + ret = set() + for item in rawlist: + fd, fam, type, laddr, raddr, status, pid = item + nt = conn_to_ntuple( + fd, + fam, + type, + laddr, + raddr, + status, + TCP_STATUSES, + pid=pid if _pid == -1 else None, + ) + ret.add(nt) + return list(ret) + + +def net_if_stats(): + """Get NIC stats (isup, duplex, speed, mtu).""" + ret = {} + rawdict = cext.net_if_stats() + for name, items in rawdict.items(): + isup, duplex, speed, mtu = items + if hasattr(_common, 'NicDuplex'): + duplex = _common.NicDuplex(duplex) + ret[name] = _common.snicstats(isup, duplex, speed, mtu, '') + return ret + + +def net_io_counters(): + """Return network I/O statistics for every network interface + installed on the system as a dict of raw tuples. + """ + return cext.net_io_counters() + + +def net_if_addrs(): + """Return the addresses associated to each NIC.""" + return cext.net_if_addrs() + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +def sensors_battery(): + """Return battery information.""" + # For constants meaning see: + # https://msdn.microsoft.com/en-us/library/windows/desktop/ + # aa373232(v=vs.85).aspx + acline_status, flags, percent, secsleft = cext.sensors_battery() + power_plugged = acline_status == 1 + no_battery = bool(flags & 128) + charging = bool(flags & 8) + + if no_battery: + return None + if power_plugged or charging: + secsleft = _common.POWER_TIME_UNLIMITED + elif secsleft == -1: + secsleft = _common.POWER_TIME_UNKNOWN + + return _common.sbattery(percent, secsleft, power_plugged) + + +# ===================================================================== +# --- other system functions +# ===================================================================== + + +_last_btime = 0 + + +def boot_time(): + """The system boot time expressed in seconds since the epoch.""" + # This dirty hack is to adjust the precision of the returned + # value which may have a 1 second fluctuation, see: + # https://github.com/giampaolo/psutil/issues/1007 + global _last_btime + ret = float(cext.boot_time()) + if abs(ret - _last_btime) <= 1: + return _last_btime + else: + _last_btime = ret + return ret + + +def users(): + """Return currently connected users as a list of namedtuples.""" + retlist = [] + rawlist = cext.users() + for item in rawlist: + user, hostname, tstamp = item + nt = _common.suser(user, None, hostname, tstamp, None) + retlist.append(nt) + return retlist + + +# ===================================================================== +# --- Windows services +# ===================================================================== + + +def win_service_iter(): + """Yields a list of WindowsService instances.""" + for name, display_name in cext.winservice_enumerate(): + yield WindowsService(name, display_name) + + +def win_service_get(name): + """Open a Windows service and return it as a WindowsService instance.""" + service = WindowsService(name, None) + service._display_name = service._query_config()['display_name'] + return service + + +class WindowsService: # noqa: PLW1641 + """Represents an installed Windows service.""" + + def __init__(self, name, display_name): + self._name = name + self._display_name = display_name + + def __str__(self): + details = f"(name={self._name!r}, display_name={self._display_name!r})" + return f"{self.__class__.__name__}{details}" + + def __repr__(self): + return f"<{self.__str__()} at {id(self)}>" + + def __eq__(self, other): + # Test for equality with another WindosService object based + # on name. + if not isinstance(other, WindowsService): + return NotImplemented + return self._name == other._name + + def __ne__(self, other): + return not self == other + + def _query_config(self): + with self._wrap_exceptions(): + display_name, binpath, username, start_type = ( + cext.winservice_query_config(self._name) + ) + # XXX - update _self.display_name? + return dict( + display_name=display_name, + binpath=binpath, + username=username, + start_type=start_type, + ) + + def _query_status(self): + with self._wrap_exceptions(): + status, pid = cext.winservice_query_status(self._name) + if pid == 0: + pid = None + return dict(status=status, pid=pid) + + @contextlib.contextmanager + def _wrap_exceptions(self): + """Ctx manager which translates bare OSError and WindowsError + exceptions into NoSuchProcess and AccessDenied. + """ + try: + yield + except OSError as err: + name = self._name + if is_permission_err(err): + msg = ( + f"service {name!r} is not querable (not enough privileges)" + ) + raise AccessDenied(pid=None, name=name, msg=msg) from err + elif err.winerror in { + cext.ERROR_INVALID_NAME, + cext.ERROR_SERVICE_DOES_NOT_EXIST, + }: + msg = f"service {name!r} does not exist" + raise NoSuchProcess(pid=None, name=name, msg=msg) from err + else: + raise + + # config query + + def name(self): + """The service name. This string is how a service is referenced + and can be passed to win_service_get() to get a new + WindowsService instance. + """ + return self._name + + def display_name(self): + """The service display name. The value is cached when this class + is instantiated. + """ + return self._display_name + + def binpath(self): + """The fully qualified path to the service binary/exe file as + a string, including command line arguments. + """ + return self._query_config()['binpath'] + + def username(self): + """The name of the user that owns this service.""" + return self._query_config()['username'] + + def start_type(self): + """A string which can either be "automatic", "manual" or + "disabled". + """ + return self._query_config()['start_type'] + + # status query + + def pid(self): + """The process PID, if any, else None. This can be passed + to Process class to control the service's process. + """ + return self._query_status()['pid'] + + def status(self): + """Service status as a string.""" + return self._query_status()['status'] + + def description(self): + """Service long description.""" + return cext.winservice_query_descr(self.name()) + + # utils + + def as_dict(self): + """Utility method retrieving all the information above as a + dictionary. + """ + d = self._query_config() + d.update(self._query_status()) + d['name'] = self.name() + d['display_name'] = self.display_name() + d['description'] = self.description() + return d + + # actions + # XXX: the necessary C bindings for start() and stop() are + # implemented but for now I prefer not to expose them. + # I may change my mind in the future. Reasons: + # - they require Administrator privileges + # - can't implement a timeout for stop() (unless by using a thread, + # which sucks) + # - would require adding ServiceAlreadyStarted and + # ServiceAlreadyStopped exceptions, adding two new APIs. + # - we might also want to have modify(), which would basically mean + # rewriting win32serviceutil.ChangeServiceConfig, which involves a + # lot of stuff (and API constants which would pollute the API), see: + # http://pyxr.sourceforge.net/PyXR/c/python24/lib/site-packages/ + # win32/lib/win32serviceutil.py.html#0175 + # - psutil is typically about "read only" monitoring stuff; + # win_service_* APIs should only be used to retrieve a service and + # check whether it's running + + # def start(self, timeout=None): + # with self._wrap_exceptions(): + # cext.winservice_start(self.name()) + # if timeout: + # giveup_at = time.time() + timeout + # while True: + # if self.status() == "running": + # return + # else: + # if time.time() > giveup_at: + # raise TimeoutExpired(timeout) + # else: + # time.sleep(.1) + + # def stop(self): + # # Note: timeout is not implemented because it's just not + # # possible, see: + # # http://stackoverflow.com/questions/11973228/ + # with self._wrap_exceptions(): + # return cext.winservice_stop(self.name()) + + +# ===================================================================== +# --- processes +# ===================================================================== + + +pids = cext.pids +pid_exists = cext.pid_exists +ppid_map = cext.ppid_map # used internally by Process.children() + + +def is_permission_err(exc): + """Return True if this is a permission error.""" + assert isinstance(exc, OSError), exc + return isinstance(exc, PermissionError) or exc.winerror in { + cext.ERROR_ACCESS_DENIED, + cext.ERROR_PRIVILEGE_NOT_HELD, + } + + +def convert_oserror(exc, pid=None, name=None): + """Convert OSError into NoSuchProcess or AccessDenied.""" + assert isinstance(exc, OSError), exc + if is_permission_err(exc): + return AccessDenied(pid=pid, name=name) + if isinstance(exc, ProcessLookupError): + return NoSuchProcess(pid=pid, name=name) + raise exc + + +def wrap_exceptions(fun): + """Decorator which converts OSError into NoSuchProcess or AccessDenied.""" + + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + try: + return fun(self, *args, **kwargs) + except OSError as err: + raise convert_oserror(err, pid=self.pid, name=self._name) from err + + return wrapper + + +def retry_error_partial_copy(fun): + """Workaround for https://github.com/giampaolo/psutil/issues/875. + See: https://stackoverflow.com/questions/4457745#4457745. + """ + + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + delay = 0.0001 + times = 33 + for _ in range(times): # retries for roughly 1 second + try: + return fun(self, *args, **kwargs) + except OSError as _: + err = _ + if err.winerror == ERROR_PARTIAL_COPY: + time.sleep(delay) + delay = min(delay * 2, 0.04) + continue + raise + msg = ( + f"{fun} retried {times} times, converted to AccessDenied as it's " + f"still returning {err}" + ) + raise AccessDenied(pid=self.pid, name=self._name, msg=msg) + + return wrapper + + +class Process: + """Wrapper class around underlying C implementation.""" + + __slots__ = ["_cache", "_name", "_ppid", "pid"] + + def __init__(self, pid): + self.pid = pid + self._name = None + self._ppid = None + + # --- oneshot() stuff + + def oneshot_enter(self): + self._proc_info.cache_activate(self) + self.exe.cache_activate(self) + + def oneshot_exit(self): + self._proc_info.cache_deactivate(self) + self.exe.cache_deactivate(self) + + @memoize_when_activated + def _proc_info(self): + """Return multiple information about this process as a + raw tuple. + """ + ret = cext.proc_info(self.pid) + assert len(ret) == len(pinfo_map) + return ret + + def name(self): + """Return process name, which on Windows is always the final + part of the executable. + """ + # This is how PIDs 0 and 4 are always represented in taskmgr + # and process-hacker. + if self.pid == 0: + return "System Idle Process" + if self.pid == 4: + return "System" + return os.path.basename(self.exe()) + + @wrap_exceptions + @memoize_when_activated + def exe(self): + if PYPY: + try: + exe = cext.proc_exe(self.pid) + except OSError as err: + # 24 = ERROR_TOO_MANY_OPEN_FILES. Not sure why this happens + # (perhaps PyPy's JIT delaying garbage collection of files?). + if err.errno == 24: + debug(f"{err!r} translated into AccessDenied") + raise AccessDenied(self.pid, self._name) from err + raise + else: + exe = cext.proc_exe(self.pid) + if exe.startswith('\\'): + return convert_dos_path(exe) + return exe # May be "Registry", "MemCompression", ... + + @wrap_exceptions + @retry_error_partial_copy + def cmdline(self): + if cext.WINVER >= cext.WINDOWS_8_1: + # PEB method detects cmdline changes but requires more + # privileges: https://github.com/giampaolo/psutil/pull/1398 + try: + return cext.proc_cmdline(self.pid, use_peb=True) + except OSError as err: + if is_permission_err(err): + return cext.proc_cmdline(self.pid, use_peb=False) + else: + raise + else: + return cext.proc_cmdline(self.pid, use_peb=True) + + @wrap_exceptions + @retry_error_partial_copy + def environ(self): + s = cext.proc_environ(self.pid) + return parse_environ_block(s) + + def ppid(self): + try: + return ppid_map()[self.pid] + except KeyError: + raise NoSuchProcess(self.pid, self._name) from None + + def _get_raw_meminfo(self): + try: + return cext.proc_memory_info(self.pid) + except OSError as err: + if is_permission_err(err): + # TODO: the C ext can probably be refactored in order + # to get this from cext.proc_info() + debug("attempting memory_info() fallback (slower)") + info = self._proc_info() + return ( + info[pinfo_map['num_page_faults']], + info[pinfo_map['peak_wset']], + info[pinfo_map['wset']], + info[pinfo_map['peak_paged_pool']], + info[pinfo_map['paged_pool']], + info[pinfo_map['peak_non_paged_pool']], + info[pinfo_map['non_paged_pool']], + info[pinfo_map['pagefile']], + info[pinfo_map['peak_pagefile']], + info[pinfo_map['mem_private']], + ) + raise + + @wrap_exceptions + def memory_info(self): + # on Windows RSS == WorkingSetSize and VSM == PagefileUsage. + # Underlying C function returns fields of PROCESS_MEMORY_COUNTERS + # struct. + t = self._get_raw_meminfo() + rss = t[2] # wset + vms = t[7] # pagefile + return pmem(*(rss, vms) + t) + + @wrap_exceptions + def memory_full_info(self): + basic_mem = self.memory_info() + uss = cext.proc_memory_uss(self.pid) + uss *= getpagesize() + return pfullmem(*basic_mem + (uss,)) + + def memory_maps(self): + try: + raw = cext.proc_memory_maps(self.pid) + except OSError as err: + # XXX - can't use wrap_exceptions decorator as we're + # returning a generator; probably needs refactoring. + raise convert_oserror(err, self.pid, self._name) from err + else: + for addr, perm, path, rss in raw: + path = convert_dos_path(path) + addr = hex(addr) + yield (addr, perm, path, rss) + + @wrap_exceptions + def kill(self): + return cext.proc_kill(self.pid) + + @wrap_exceptions + def send_signal(self, sig): + if sig == signal.SIGTERM: + cext.proc_kill(self.pid) + elif sig in {signal.CTRL_C_EVENT, signal.CTRL_BREAK_EVENT}: + os.kill(self.pid, sig) + else: + msg = ( + "only SIGTERM, CTRL_C_EVENT and CTRL_BREAK_EVENT signals " + "are supported on Windows" + ) + raise ValueError(msg) + + @wrap_exceptions + def wait(self, timeout=None): + if timeout is None: + cext_timeout = cext.INFINITE + else: + # WaitForSingleObject() expects time in milliseconds. + cext_timeout = int(timeout * 1000) + + timer = getattr(time, 'monotonic', time.time) + stop_at = timer() + timeout if timeout is not None else None + + try: + # Exit code is supposed to come from GetExitCodeProcess(). + # May also be None if OpenProcess() failed with + # ERROR_INVALID_PARAMETER, meaning PID is already gone. + exit_code = cext.proc_wait(self.pid, cext_timeout) + except cext.TimeoutExpired as err: + # WaitForSingleObject() returned WAIT_TIMEOUT. Just raise. + raise TimeoutExpired(timeout, self.pid, self._name) from err + except cext.TimeoutAbandoned: + # WaitForSingleObject() returned WAIT_ABANDONED, see: + # https://github.com/giampaolo/psutil/issues/1224 + # We'll just rely on the internal polling and return None + # when the PID disappears. Subprocess module does the same + # (return None): + # https://github.com/python/cpython/blob/ + # be50a7b627d0aa37e08fa8e2d5568891f19903ce/ + # Lib/subprocess.py#L1193-L1194 + exit_code = None + + # At this point WaitForSingleObject() returned WAIT_OBJECT_0, + # meaning the process is gone. Stupidly there are cases where + # its PID may still stick around so we do a further internal + # polling. + delay = 0.0001 + while True: + if not pid_exists(self.pid): + return exit_code + if stop_at and timer() >= stop_at: + raise TimeoutExpired(timeout, pid=self.pid, name=self._name) + time.sleep(delay) + delay = min(delay * 2, 0.04) # incremental delay + + @wrap_exceptions + def username(self): + if self.pid in {0, 4}: + return 'NT AUTHORITY\\SYSTEM' + domain, user = cext.proc_username(self.pid) + return f"{domain}\\{user}" + + @wrap_exceptions + def create_time(self, fast_only=False): + # Note: proc_times() not put under oneshot() 'cause create_time() + # is already cached by the main Process class. + try: + _user, _system, created = cext.proc_times(self.pid) + return created + except OSError as err: + if is_permission_err(err): + if fast_only: + raise + debug("attempting create_time() fallback (slower)") + return self._proc_info()[pinfo_map['create_time']] + raise + + @wrap_exceptions + def num_threads(self): + return self._proc_info()[pinfo_map['num_threads']] + + @wrap_exceptions + def threads(self): + rawlist = cext.proc_threads(self.pid) + retlist = [] + for thread_id, utime, stime in rawlist: + ntuple = _common.pthread(thread_id, utime, stime) + retlist.append(ntuple) + return retlist + + @wrap_exceptions + def cpu_times(self): + try: + user, system, _created = cext.proc_times(self.pid) + except OSError as err: + if not is_permission_err(err): + raise + debug("attempting cpu_times() fallback (slower)") + info = self._proc_info() + user = info[pinfo_map['user_time']] + system = info[pinfo_map['kernel_time']] + # Children user/system times are not retrievable (set to 0). + return _common.pcputimes(user, system, 0.0, 0.0) + + @wrap_exceptions + def suspend(self): + cext.proc_suspend_or_resume(self.pid, True) + + @wrap_exceptions + def resume(self): + cext.proc_suspend_or_resume(self.pid, False) + + @wrap_exceptions + @retry_error_partial_copy + def cwd(self): + if self.pid in {0, 4}: + raise AccessDenied(self.pid, self._name) + # return a normalized pathname since the native C function appends + # "\\" at the and of the path + path = cext.proc_cwd(self.pid) + return os.path.normpath(path) + + @wrap_exceptions + def open_files(self): + if self.pid in {0, 4}: + return [] + ret = set() + # Filenames come in in native format like: + # "\Device\HarddiskVolume1\Windows\systemew\file.txt" + # Convert the first part in the corresponding drive letter + # (e.g. "C:\") by using Windows's QueryDosDevice() + raw_file_names = cext.proc_open_files(self.pid) + for file in raw_file_names: + file = convert_dos_path(file) + if isfile_strict(file): + ntuple = _common.popenfile(file, -1) + ret.add(ntuple) + return list(ret) + + @wrap_exceptions + def net_connections(self, kind='inet'): + return net_connections(kind, _pid=self.pid) + + @wrap_exceptions + def nice_get(self): + value = cext.proc_priority_get(self.pid) + value = Priority(value) + return value + + @wrap_exceptions + def nice_set(self, value): + return cext.proc_priority_set(self.pid, value) + + @wrap_exceptions + def ionice_get(self): + ret = cext.proc_io_priority_get(self.pid) + ret = IOPriority(ret) + return ret + + @wrap_exceptions + def ionice_set(self, ioclass, value): + if value: + msg = "value argument not accepted on Windows" + raise TypeError(msg) + if ioclass not in { + IOPriority.IOPRIO_VERYLOW, + IOPriority.IOPRIO_LOW, + IOPriority.IOPRIO_NORMAL, + IOPriority.IOPRIO_HIGH, + }: + msg = f"{ioclass} is not a valid priority" + raise ValueError(msg) + cext.proc_io_priority_set(self.pid, ioclass) + + @wrap_exceptions + def io_counters(self): + try: + ret = cext.proc_io_counters(self.pid) + except OSError as err: + if not is_permission_err(err): + raise + debug("attempting io_counters() fallback (slower)") + info = self._proc_info() + ret = ( + info[pinfo_map['io_rcount']], + info[pinfo_map['io_wcount']], + info[pinfo_map['io_rbytes']], + info[pinfo_map['io_wbytes']], + info[pinfo_map['io_count_others']], + info[pinfo_map['io_bytes_others']], + ) + return pio(*ret) + + @wrap_exceptions + def status(self): + suspended = cext.proc_is_suspended(self.pid) + if suspended: + return _common.STATUS_STOPPED + else: + return _common.STATUS_RUNNING + + @wrap_exceptions + def cpu_affinity_get(self): + def from_bitmask(x): + return [i for i in range(64) if (1 << i) & x] + + bitmask = cext.proc_cpu_affinity_get(self.pid) + return from_bitmask(bitmask) + + @wrap_exceptions + def cpu_affinity_set(self, value): + def to_bitmask(ls): + if not ls: + msg = f"invalid argument {ls!r}" + raise ValueError(msg) + out = 0 + for b in ls: + out |= 2**b + return out + + # SetProcessAffinityMask() states that ERROR_INVALID_PARAMETER + # is returned for an invalid CPU but this seems not to be true, + # therefore we check CPUs validy beforehand. + allcpus = list(range(len(per_cpu_times()))) + for cpu in value: + if cpu not in allcpus: + if not isinstance(cpu, int): + msg = f"invalid CPU {cpu!r}; an integer is required" + raise TypeError(msg) + msg = f"invalid CPU {cpu!r}" + raise ValueError(msg) + + bitmask = to_bitmask(value) + cext.proc_cpu_affinity_set(self.pid, bitmask) + + @wrap_exceptions + def num_handles(self): + try: + return cext.proc_num_handles(self.pid) + except OSError as err: + if is_permission_err(err): + debug("attempting num_handles() fallback (slower)") + return self._proc_info()[pinfo_map['num_handles']] + raise + + @wrap_exceptions + def num_ctx_switches(self): + ctx_switches = self._proc_info()[pinfo_map['ctx_switches']] + # only voluntary ctx switches are supported + return _common.pctxsw(ctx_switches, 0) diff --git a/lib/python3.12/site-packages/psutil/tests/__init__.py b/lib/python3.12/site-packages/psutil/tests/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..5d4b3abb7965010010f1810b6007bc912e1887bf --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/__init__.py @@ -0,0 +1,2025 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Test utilities.""" + + +import atexit +import contextlib +import ctypes +import enum +import errno +import functools +import gc +import importlib +import ipaddress +import os +import platform +import random +import re +import select +import shlex +import shutil +import signal +import socket +import stat +import subprocess +import sys +import tempfile +import textwrap +import threading +import time +import unittest +import warnings +from socket import AF_INET +from socket import AF_INET6 +from socket import SOCK_STREAM + + +try: + import pytest +except ImportError: + pytest = None + +import psutil +from psutil import AIX +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil._common import bytes2human +from psutil._common import debug +from psutil._common import memoize +from psutil._common import print_color +from psutil._common import supports_ipv6 + + +if POSIX: + from psutil._psposix import wait_pid + + +# fmt: off +__all__ = [ + # constants + 'DEVNULL', 'GLOBAL_TIMEOUT', 'TOLERANCE_SYS_MEM', 'NO_RETRIES', + 'PYPY', 'PYTHON_EXE', 'PYTHON_EXE_ENV', 'ROOT_DIR', 'SCRIPTS_DIR', + 'TESTFN_PREFIX', 'UNICODE_SUFFIX', 'INVALID_UNICODE_SUFFIX', + 'CI_TESTING', 'VALID_PROC_STATUSES', 'TOLERANCE_DISK_USAGE', 'IS_64BIT', + "HAS_CPU_AFFINITY", "HAS_CPU_FREQ", "HAS_ENVIRON", "HAS_PROC_IO_COUNTERS", + "HAS_IONICE", "HAS_MEMORY_MAPS", "HAS_PROC_CPU_NUM", "HAS_RLIMIT", + "HAS_SENSORS_BATTERY", "HAS_BATTERY", "HAS_SENSORS_FANS", + "HAS_SENSORS_TEMPERATURES", "HAS_NET_CONNECTIONS_UNIX", "MACOS_11PLUS", + "MACOS_12PLUS", "COVERAGE", 'AARCH64', "PYTEST_PARALLEL", + # subprocesses + 'pyrun', 'terminate', 'reap_children', 'spawn_testproc', 'spawn_zombie', + 'spawn_children_pair', + # threads + 'ThreadTask', + # test utils + 'unittest', 'skip_on_access_denied', 'skip_on_not_implemented', + 'retry_on_failure', 'TestMemoryLeak', 'PsutilTestCase', + 'process_namespace', 'system_namespace', 'print_sysinfo', + 'is_win_secure_system_proc', 'fake_pytest', + # fs utils + 'chdir', 'safe_rmpath', 'create_py_exe', 'create_c_exe', 'get_testfn', + # os + 'get_winver', 'kernel_version', + # sync primitives + 'call_until', 'wait_for_pid', 'wait_for_file', + # network + 'check_net_address', 'filter_proc_net_connections', + 'get_free_port', 'bind_socket', 'bind_unix_socket', 'tcp_socketpair', + 'unix_socketpair', 'create_sockets', + # compat + 'reload_module', 'import_module_by_path', + # others + 'warn', 'copyload_shared_lib', 'is_namedtuple', +] +# fmt: on + + +# =================================================================== +# --- constants +# =================================================================== + +# --- platforms + +PYPY = '__pypy__' in sys.builtin_module_names +# whether we're running this test suite on a Continuous Integration service +GITHUB_ACTIONS = 'GITHUB_ACTIONS' in os.environ or 'CIBUILDWHEEL' in os.environ +CI_TESTING = GITHUB_ACTIONS +COVERAGE = 'COVERAGE_RUN' in os.environ +PYTEST_PARALLEL = "PYTEST_XDIST_WORKER" in os.environ # `make test-parallel` +# are we a 64 bit process? +IS_64BIT = sys.maxsize > 2**32 +AARCH64 = platform.machine() == "aarch64" + + +@memoize +def macos_version(): + version_str = platform.mac_ver()[0] + version = tuple(map(int, version_str.split(".")[:2])) + if version == (10, 16): + # When built against an older macOS SDK, Python will report + # macOS 10.16 instead of the real version. + version_str = subprocess.check_output( + [ + sys.executable, + "-sS", + "-c", + "import platform; print(platform.mac_ver()[0])", + ], + env={"SYSTEM_VERSION_COMPAT": "0"}, + universal_newlines=True, + ) + version = tuple(map(int, version_str.split(".")[:2])) + return version + + +if MACOS: + MACOS_11PLUS = macos_version() > (10, 15) + MACOS_12PLUS = macos_version() >= (12, 0) +else: + MACOS_11PLUS = False + MACOS_12PLUS = False + + +# --- configurable defaults + +# how many times retry_on_failure() decorator will retry +NO_RETRIES = 10 +# bytes tolerance for system-wide related tests +TOLERANCE_SYS_MEM = 5 * 1024 * 1024 # 5MB +TOLERANCE_DISK_USAGE = 10 * 1024 * 1024 # 10MB +# the timeout used in functions which have to wait +GLOBAL_TIMEOUT = 5 +# be more tolerant if we're on CI in order to avoid false positives +if CI_TESTING: + NO_RETRIES *= 3 + GLOBAL_TIMEOUT *= 3 + TOLERANCE_SYS_MEM *= 4 + TOLERANCE_DISK_USAGE *= 3 + +# --- file names + +# Disambiguate TESTFN for parallel testing. +if os.name == 'java': + # Jython disallows @ in module names + TESTFN_PREFIX = f"$psutil-{os.getpid()}-" +else: + TESTFN_PREFIX = f"@psutil-{os.getpid()}-" +UNICODE_SUFFIX = "-ƒőő" +# An invalid unicode string. +INVALID_UNICODE_SUFFIX = b"f\xc0\x80".decode('utf8', 'surrogateescape') +ASCII_FS = sys.getfilesystemencoding().lower() in {"ascii", "us-ascii"} + +# --- paths + +ROOT_DIR = os.path.realpath( + os.path.join(os.path.dirname(__file__), '..', '..') +) +SCRIPTS_DIR = os.environ.get( + "PSUTIL_SCRIPTS_DIR", os.path.join(ROOT_DIR, 'scripts') +) +HERE = os.path.realpath(os.path.dirname(__file__)) + +# --- support + +HAS_CPU_AFFINITY = hasattr(psutil.Process, "cpu_affinity") +HAS_CPU_FREQ = hasattr(psutil, "cpu_freq") +HAS_ENVIRON = hasattr(psutil.Process, "environ") +HAS_GETLOADAVG = hasattr(psutil, "getloadavg") +HAS_IONICE = hasattr(psutil.Process, "ionice") +HAS_MEMORY_MAPS = hasattr(psutil.Process, "memory_maps") +HAS_NET_CONNECTIONS_UNIX = POSIX and not SUNOS +HAS_NET_IO_COUNTERS = hasattr(psutil, "net_io_counters") +HAS_PROC_CPU_NUM = hasattr(psutil.Process, "cpu_num") +HAS_PROC_IO_COUNTERS = hasattr(psutil.Process, "io_counters") +HAS_RLIMIT = hasattr(psutil.Process, "rlimit") +HAS_SENSORS_BATTERY = hasattr(psutil, "sensors_battery") +try: + HAS_BATTERY = HAS_SENSORS_BATTERY and bool(psutil.sensors_battery()) +except Exception: # noqa: BLE001 + HAS_BATTERY = False +HAS_SENSORS_FANS = hasattr(psutil, "sensors_fans") +HAS_SENSORS_TEMPERATURES = hasattr(psutil, "sensors_temperatures") +HAS_THREADS = hasattr(psutil.Process, "threads") +SKIP_SYSCONS = (MACOS or AIX) and os.getuid() != 0 + +# --- misc + + +def _get_py_exe(): + def attempt(exe): + try: + subprocess.check_call( + [exe, "-V"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + except subprocess.CalledProcessError: + return None + else: + return exe + + env = os.environ.copy() + + # On Windows, starting with python 3.7, virtual environments use a + # venv launcher startup process. This does not play well when + # counting spawned processes, or when relying on the PID of the + # spawned process to do some checks, e.g. connections check per PID. + # Let's use the base python in this case. + base = getattr(sys, "_base_executable", None) + if WINDOWS and sys.version_info >= (3, 7) and base is not None: + # We need to set __PYVENV_LAUNCHER__ to sys.executable for the + # base python executable to know about the environment. + env["__PYVENV_LAUNCHER__"] = sys.executable + return base, env + elif GITHUB_ACTIONS: + return sys.executable, env + elif MACOS: + exe = ( + attempt(sys.executable) + or attempt(os.path.realpath(sys.executable)) + or attempt( + shutil.which("python{}.{}".format(*sys.version_info[:2])) + ) + or attempt(psutil.Process().exe()) + ) + if not exe: + raise ValueError("can't find python exe real abspath") + return exe, env + else: + exe = os.path.realpath(sys.executable) + assert os.path.exists(exe), exe + return exe, env + + +PYTHON_EXE, PYTHON_EXE_ENV = _get_py_exe() +DEVNULL = open(os.devnull, 'r+') # noqa: SIM115 +atexit.register(DEVNULL.close) + +VALID_PROC_STATUSES = [ + getattr(psutil, x) for x in dir(psutil) if x.startswith('STATUS_') +] +AF_UNIX = getattr(socket, "AF_UNIX", object()) + +_subprocesses_started = set() +_pids_started = set() + + +# =================================================================== +# --- threads +# =================================================================== + + +class ThreadTask(threading.Thread): + """A thread task which does nothing expect staying alive.""" + + def __init__(self): + super().__init__() + self._running = False + self._interval = 0.001 + self._flag = threading.Event() + + def __repr__(self): + name = self.__class__.__name__ + return f"<{name} running={self._running} at {id(self):#x}>" + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args, **kwargs): + self.stop() + + def start(self): + """Start thread and keep it running until an explicit + stop() request. Polls for shutdown every 'timeout' seconds. + """ + if self._running: + raise ValueError("already started") + threading.Thread.start(self) + self._flag.wait() + + def run(self): + self._running = True + self._flag.set() + while self._running: + time.sleep(self._interval) + + def stop(self): + """Stop thread execution and and waits until it is stopped.""" + if not self._running: + raise ValueError("already stopped") + self._running = False + self.join() + + +# =================================================================== +# --- subprocesses +# =================================================================== + + +def _reap_children_on_err(fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except Exception: + reap_children() + raise + + return wrapper + + +@_reap_children_on_err +def spawn_testproc(cmd=None, **kwds): + """Create a python subprocess which does nothing for some secs and + return it as a subprocess.Popen instance. + If "cmd" is specified that is used instead of python. + By default stdin and stdout are redirected to /dev/null. + It also attempts to make sure the process is in a reasonably + initialized state. + The process is registered for cleanup on reap_children(). + """ + kwds.setdefault("stdin", DEVNULL) + kwds.setdefault("stdout", DEVNULL) + kwds.setdefault("cwd", os.getcwd()) + kwds.setdefault("env", PYTHON_EXE_ENV) + if WINDOWS: + # Prevents the subprocess to open error dialogs. This will also + # cause stderr to be suppressed, which is suboptimal in order + # to debug broken tests. + CREATE_NO_WINDOW = 0x8000000 + kwds.setdefault("creationflags", CREATE_NO_WINDOW) + if cmd is None: + testfn = get_testfn(dir=os.getcwd()) + try: + safe_rmpath(testfn) + pyline = ( + "import time;" + f"open(r'{testfn}', 'w').close();" + "[time.sleep(0.1) for x in range(100)];" # 10 secs + ) + cmd = [PYTHON_EXE, "-c", pyline] + sproc = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(sproc) + wait_for_file(testfn, delete=True, empty=True) + finally: + safe_rmpath(testfn) + else: + sproc = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(sproc) + wait_for_pid(sproc.pid) + return sproc + + +@_reap_children_on_err +def spawn_children_pair(): + """Create a subprocess which creates another one as in: + A (us) -> B (child) -> C (grandchild). + Return a (child, grandchild) tuple. + The 2 processes are fully initialized and will live for 60 secs + and are registered for cleanup on reap_children(). + """ + tfile = None + testfn = get_testfn(dir=os.getcwd()) + try: + s = textwrap.dedent(f"""\ + import subprocess, os, sys, time + s = "import os, time;" + s += "f = open('{os.path.basename(testfn)}', 'w');" + s += "f.write(str(os.getpid()));" + s += "f.close();" + s += "[time.sleep(0.1) for x in range(100 * 6)];" + p = subprocess.Popen([r'{PYTHON_EXE}', '-c', s]) + p.wait() + """) + # On Windows if we create a subprocess with CREATE_NO_WINDOW flag + # set (which is the default) a "conhost.exe" extra process will be + # spawned as a child. We don't want that. + if WINDOWS: + subp, tfile = pyrun(s, creationflags=0) + else: + subp, tfile = pyrun(s) + child = psutil.Process(subp.pid) + grandchild_pid = int(wait_for_file(testfn, delete=True, empty=False)) + _pids_started.add(grandchild_pid) + grandchild = psutil.Process(grandchild_pid) + return (child, grandchild) + finally: + safe_rmpath(testfn) + if tfile is not None: + safe_rmpath(tfile) + + +def spawn_zombie(): + """Create a zombie process and return a (parent, zombie) process tuple. + In order to kill the zombie parent must be terminate()d first, then + zombie must be wait()ed on. + """ + assert psutil.POSIX + unix_file = get_testfn() + src = textwrap.dedent(f"""\ + import os, sys, time, socket, contextlib + child_pid = os.fork() + if child_pid > 0: + time.sleep(3000) + else: + # this is the zombie process + with socket.socket(socket.AF_UNIX) as s: + s.connect('{unix_file}') + pid = bytes(str(os.getpid()), 'ascii') + s.sendall(pid) + """) + tfile = None + sock = bind_unix_socket(unix_file) + try: + sock.settimeout(GLOBAL_TIMEOUT) + parent, tfile = pyrun(src) + conn, _ = sock.accept() + try: + select.select([conn.fileno()], [], [], GLOBAL_TIMEOUT) + zpid = int(conn.recv(1024)) + _pids_started.add(zpid) + zombie = psutil.Process(zpid) + call_until(lambda: zombie.status() == psutil.STATUS_ZOMBIE) + return (parent, zombie) + finally: + conn.close() + finally: + sock.close() + safe_rmpath(unix_file) + if tfile is not None: + safe_rmpath(tfile) + + +@_reap_children_on_err +def pyrun(src, **kwds): + """Run python 'src' code string in a separate interpreter. + Returns a subprocess.Popen instance and the test file where the source + code was written. + """ + kwds.setdefault("stdout", None) + kwds.setdefault("stderr", None) + srcfile = get_testfn() + try: + with open(srcfile, "w") as f: + f.write(src) + subp = spawn_testproc([PYTHON_EXE, f.name], **kwds) + wait_for_pid(subp.pid) + return (subp, srcfile) + except Exception: + safe_rmpath(srcfile) + raise + + +@_reap_children_on_err +def sh(cmd, **kwds): + """Run cmd in a subprocess and return its output. + raises RuntimeError on error. + """ + # Prevents subprocess to open error dialogs in case of error. + flags = 0x8000000 if WINDOWS else 0 + kwds.setdefault("stdout", subprocess.PIPE) + kwds.setdefault("stderr", subprocess.PIPE) + kwds.setdefault("universal_newlines", True) + kwds.setdefault("creationflags", flags) + if isinstance(cmd, str): + cmd = shlex.split(cmd) + p = subprocess.Popen(cmd, **kwds) + _subprocesses_started.add(p) + stdout, stderr = p.communicate(timeout=GLOBAL_TIMEOUT) + if p.returncode != 0: + raise RuntimeError(stdout + stderr) + if stderr: + warn(stderr) + if stdout.endswith('\n'): + stdout = stdout[:-1] + return stdout + + +def terminate(proc_or_pid, sig=signal.SIGTERM, wait_timeout=GLOBAL_TIMEOUT): + """Terminate a process and wait() for it. + Process can be a PID or an instance of psutil.Process(), + subprocess.Popen() or psutil.Popen(). + If it's a subprocess.Popen() or psutil.Popen() instance also closes + its stdin / stdout / stderr fds. + PID is wait()ed even if the process is already gone (kills zombies). + Does nothing if the process does not exist. + Return process exit status. + """ + + def wait(proc, timeout): + proc.wait(timeout) + if WINDOWS and isinstance(proc, subprocess.Popen): + # Otherwise PID may still hang around. + try: + return psutil.Process(proc.pid).wait(timeout) + except psutil.NoSuchProcess: + pass + + def sendsig(proc, sig): + # XXX: otherwise the build hangs for some reason. + if MACOS and GITHUB_ACTIONS: + sig = signal.SIGKILL + # If the process received SIGSTOP, SIGCONT is necessary first, + # otherwise SIGTERM won't work. + if POSIX and sig != signal.SIGKILL: + proc.send_signal(signal.SIGCONT) + proc.send_signal(sig) + + def term_subprocess_proc(proc, timeout): + try: + sendsig(proc, sig) + except ProcessLookupError: + pass + except OSError as err: + if WINDOWS and err.winerror == 6: # "invalid handle" + pass + raise + return wait(proc, timeout) + + def term_psutil_proc(proc, timeout): + try: + sendsig(proc, sig) + except psutil.NoSuchProcess: + pass + return wait(proc, timeout) + + def term_pid(pid, timeout): + try: + proc = psutil.Process(pid) + except psutil.NoSuchProcess: + # Needed to kill zombies. + if POSIX: + return wait_pid(pid, timeout) + else: + return term_psutil_proc(proc, timeout) + + def flush_popen(proc): + if proc.stdout: + proc.stdout.close() + if proc.stderr: + proc.stderr.close() + # Flushing a BufferedWriter may raise an error. + if proc.stdin: + proc.stdin.close() + + p = proc_or_pid + try: + if isinstance(p, int): + return term_pid(p, wait_timeout) + elif isinstance(p, (psutil.Process, psutil.Popen)): + return term_psutil_proc(p, wait_timeout) + elif isinstance(p, subprocess.Popen): + return term_subprocess_proc(p, wait_timeout) + else: + raise TypeError(f"wrong type {p!r}") + finally: + if isinstance(p, (subprocess.Popen, psutil.Popen)): + flush_popen(p) + pid = p if isinstance(p, int) else p.pid + assert not psutil.pid_exists(pid), pid + + +def reap_children(recursive=False): + """Terminate and wait() any subprocess started by this test suite + and any children currently running, ensuring that no processes stick + around to hog resources. + If recursive is True it also tries to terminate and wait() + all grandchildren started by this process. + """ + # Get the children here before terminating them, as in case of + # recursive=True we don't want to lose the intermediate reference + # pointing to the grandchildren. + children = psutil.Process().children(recursive=recursive) + + # Terminate subprocess.Popen. + while _subprocesses_started: + subp = _subprocesses_started.pop() + terminate(subp) + + # Collect started pids. + while _pids_started: + pid = _pids_started.pop() + terminate(pid) + + # Terminate children. + if children: + for p in children: + terminate(p, wait_timeout=None) + _, alive = psutil.wait_procs(children, timeout=GLOBAL_TIMEOUT) + for p in alive: + warn(f"couldn't terminate process {p!r}; attempting kill()") + terminate(p, sig=signal.SIGKILL) + + +# =================================================================== +# --- OS +# =================================================================== + + +def kernel_version(): + """Return a tuple such as (2, 6, 36).""" + if not POSIX: + raise NotImplementedError("not POSIX") + s = "" + uname = os.uname()[2] + for c in uname: + if c.isdigit() or c == '.': + s += c + else: + break + if not s: + raise ValueError(f"can't parse {uname!r}") + minor = 0 + micro = 0 + nums = s.split('.') + major = int(nums[0]) + if len(nums) >= 2: + minor = int(nums[1]) + if len(nums) >= 3: + micro = int(nums[2]) + return (major, minor, micro) + + +def get_winver(): + if not WINDOWS: + raise NotImplementedError("not WINDOWS") + wv = sys.getwindowsversion() + sp = wv.service_pack_major or 0 + return (wv[0], wv[1], sp) + + +# =================================================================== +# --- sync primitives +# =================================================================== + + +class retry: + """A retry decorator.""" + + def __init__( + self, + exception=Exception, + timeout=None, + retries=None, + interval=0.001, + logfun=None, + ): + if timeout and retries: + raise ValueError("timeout and retries args are mutually exclusive") + self.exception = exception + self.timeout = timeout + self.retries = retries + self.interval = interval + self.logfun = logfun + + def __iter__(self): + if self.timeout: + stop_at = time.time() + self.timeout + while time.time() < stop_at: + yield + elif self.retries: + for _ in range(self.retries): + yield + else: + while True: + yield + + def sleep(self): + if self.interval is not None: + time.sleep(self.interval) + + def __call__(self, fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + exc = None + for _ in self: + try: + return fun(*args, **kwargs) + except self.exception as _: + exc = _ + if self.logfun is not None: + self.logfun(exc) + self.sleep() + continue + + raise exc + + # This way the user of the decorated function can change config + # parameters. + wrapper.decorator = self + return wrapper + + +@retry( + exception=psutil.NoSuchProcess, + logfun=None, + timeout=GLOBAL_TIMEOUT, + interval=0.001, +) +def wait_for_pid(pid): + """Wait for pid to show up in the process list then return. + Used in the test suite to give time the sub process to initialize. + """ + if pid not in psutil.pids(): + raise psutil.NoSuchProcess(pid) + psutil.Process(pid) + + +@retry( + exception=(FileNotFoundError, AssertionError), + logfun=None, + timeout=GLOBAL_TIMEOUT, + interval=0.001, +) +def wait_for_file(fname, delete=True, empty=False): + """Wait for a file to be written on disk with some content.""" + with open(fname, "rb") as f: + data = f.read() + if not empty: + assert data + if delete: + safe_rmpath(fname) + return data + + +@retry( + exception=AssertionError, + logfun=None, + timeout=GLOBAL_TIMEOUT, + interval=0.001, +) +def call_until(fun): + """Keep calling function until it evaluates to True.""" + ret = fun() + assert ret + return ret + + +# =================================================================== +# --- fs +# =================================================================== + + +def safe_rmpath(path): + """Convenience function for removing temporary test files or dirs.""" + + def retry_fun(fun): + # On Windows it could happen that the file or directory has + # open handles or references preventing the delete operation + # to succeed immediately, so we retry for a while. See: + # https://bugs.python.org/issue33240 + stop_at = time.time() + GLOBAL_TIMEOUT + while time.time() < stop_at: + try: + return fun() + except FileNotFoundError: + pass + except OSError as _: + err = _ + warn(f"ignoring {err}") + time.sleep(0.01) + raise err + + try: + st = os.stat(path) + if stat.S_ISDIR(st.st_mode): + fun = functools.partial(shutil.rmtree, path) + else: + fun = functools.partial(os.remove, path) + if POSIX: + fun() + else: + retry_fun(fun) + except FileNotFoundError: + pass + + +def safe_mkdir(dir): + """Convenience function for creating a directory.""" + try: + os.mkdir(dir) + except FileExistsError: + pass + + +@contextlib.contextmanager +def chdir(dirname): + """Context manager which temporarily changes the current directory.""" + curdir = os.getcwd() + try: + os.chdir(dirname) + yield + finally: + os.chdir(curdir) + + +def create_py_exe(path): + """Create a Python executable file in the given location.""" + assert not os.path.exists(path), path + atexit.register(safe_rmpath, path) + shutil.copyfile(PYTHON_EXE, path) + if POSIX: + st = os.stat(path) + os.chmod(path, st.st_mode | stat.S_IEXEC) + return path + + +def create_c_exe(path, c_code=None): + """Create a compiled C executable in the given location.""" + assert not os.path.exists(path), path + if not shutil.which("gcc"): + raise pytest.skip("gcc is not installed") + if c_code is None: + c_code = textwrap.dedent(""" + #include + int main() { + pause(); + return 1; + } + """) + else: + assert isinstance(c_code, str), c_code + + atexit.register(safe_rmpath, path) + with open(get_testfn(suffix='.c'), "w") as f: + f.write(c_code) + try: + subprocess.check_call(["gcc", f.name, "-o", path]) + finally: + safe_rmpath(f.name) + return path + + +def get_testfn(suffix="", dir=None): + """Return an absolute pathname of a file or dir that did not + exist at the time this call is made. Also schedule it for safe + deletion at interpreter exit. It's technically racy but probably + not really due to the time variant. + """ + while True: + name = tempfile.mktemp(prefix=TESTFN_PREFIX, suffix=suffix, dir=dir) + if not os.path.exists(name): # also include dirs + path = os.path.realpath(name) # needed for OSX + atexit.register(safe_rmpath, path) + return path + + +# =================================================================== +# --- testing +# =================================================================== + + +class fake_pytest: + """A class that mimics some basic pytest APIs. This is meant for + when unit tests are run in production, where pytest may not be + installed. Still, the user can test psutil installation via: + + $ python3 -m psutil.tests + """ + + @staticmethod + def main(*args, **kw): # noqa: ARG004 + """Mimics pytest.main(). It has the same effect as running + `python3 -m unittest -v` from the project root directory. + """ + suite = unittest.TestLoader().discover(HERE) + unittest.TextTestRunner(verbosity=2).run(suite) + warnings.warn( + "Fake pytest module was used. Test results may be inaccurate.", + UserWarning, + stacklevel=1, + ) + return suite + + @staticmethod + def raises(exc, match=None): + """Mimics `pytest.raises`.""" + + class ExceptionInfo: + _exc = None + + @property + def value(self): + return self._exc + + @contextlib.contextmanager + def context(exc, match=None): + einfo = ExceptionInfo() + try: + yield einfo + except exc as err: + if match and not re.search(match, str(err)): + msg = f'"{match}" does not match "{err}"' + raise AssertionError(msg) + einfo._exc = err + else: + raise AssertionError(f"{exc!r} not raised") + + return context(exc, match=match) + + @staticmethod + def warns(warning, match=None): + """Mimics `pytest.warns`.""" + if match: + return unittest.TestCase().assertWarnsRegex(warning, match) + return unittest.TestCase().assertWarns(warning) + + @staticmethod + def skip(reason=""): + """Mimics `unittest.SkipTest`.""" + raise unittest.SkipTest(reason) + + class mark: + + @staticmethod + def skipif(condition, reason=""): + """Mimics `@pytest.mark.skipif` decorator.""" + return unittest.skipIf(condition, reason) + + class xdist_group: + """Mimics `@pytest.mark.xdist_group` decorator (no-op).""" + + def __init__(self, name=None): + pass + + def __call__(self, cls_or_meth): + return cls_or_meth + + +if pytest is None: + pytest = fake_pytest + + +class PsutilTestCase(unittest.TestCase): + """Test class providing auto-cleanup wrappers on top of process + test utilities. All test classes should derive from this one, even + if we use pytest. + """ + + def get_testfn(self, suffix="", dir=None): + fname = get_testfn(suffix=suffix, dir=dir) + self.addCleanup(safe_rmpath, fname) + return fname + + def spawn_testproc(self, *args, **kwds): + sproc = spawn_testproc(*args, **kwds) + self.addCleanup(terminate, sproc) + return sproc + + def spawn_children_pair(self): + child1, child2 = spawn_children_pair() + self.addCleanup(terminate, child2) + self.addCleanup(terminate, child1) # executed first + return (child1, child2) + + def spawn_zombie(self): + parent, zombie = spawn_zombie() + self.addCleanup(terminate, zombie) + self.addCleanup(terminate, parent) # executed first + return (parent, zombie) + + def pyrun(self, *args, **kwds): + sproc, srcfile = pyrun(*args, **kwds) + self.addCleanup(safe_rmpath, srcfile) + self.addCleanup(terminate, sproc) # executed first + return sproc + + def _check_proc_exc(self, proc, exc): + assert isinstance(exc, psutil.Error) + assert exc.pid == proc.pid + assert exc.name == proc._name + if exc.name: + assert exc.name + if isinstance(exc, psutil.ZombieProcess): + assert exc.ppid == proc._ppid + if exc.ppid is not None: + assert exc.ppid >= 0 + str(exc) + repr(exc) + + def assertPidGone(self, pid): + with pytest.raises(psutil.NoSuchProcess) as cm: + try: + psutil.Process(pid) + except psutil.ZombieProcess: + raise AssertionError("wasn't supposed to raise ZombieProcess") + assert cm.value.pid == pid + assert cm.value.name is None + assert not psutil.pid_exists(pid), pid + assert pid not in psutil.pids() + assert pid not in [x.pid for x in psutil.process_iter()] + + def assertProcessGone(self, proc): + self.assertPidGone(proc.pid) + ns = process_namespace(proc) + for fun, name in ns.iter(ns.all, clear_cache=True): + with self.subTest(proc=proc, name=name): + try: + ret = fun() + except psutil.ZombieProcess: + raise + except psutil.NoSuchProcess as exc: + self._check_proc_exc(proc, exc) + else: + msg = ( + f"Process.{name}() didn't raise NSP and returned" + f" {ret!r}" + ) + raise AssertionError(msg) + proc.wait(timeout=0) # assert not raise TimeoutExpired + + def assertProcessZombie(self, proc): + # A zombie process should always be instantiable. + clone = psutil.Process(proc.pid) + # Cloned zombie on Open/NetBSD has null creation time, see: + # https://github.com/giampaolo/psutil/issues/2287 + assert proc == clone + if not (OPENBSD or NETBSD): + assert hash(proc) == hash(clone) + # Its status always be querable. + assert proc.status() == psutil.STATUS_ZOMBIE + # It should be considered 'running'. + assert proc.is_running() + assert psutil.pid_exists(proc.pid) + # as_dict() shouldn't crash. + proc.as_dict() + # It should show up in pids() and process_iter(). + assert proc.pid in psutil.pids() + assert proc.pid in [x.pid for x in psutil.process_iter()] + psutil._pmap = {} + assert proc.pid in [x.pid for x in psutil.process_iter()] + # Call all methods. + ns = process_namespace(proc) + for fun, name in ns.iter(ns.all, clear_cache=True): + with self.subTest(proc=proc, name=name): + try: + fun() + except (psutil.ZombieProcess, psutil.AccessDenied) as exc: + self._check_proc_exc(proc, exc) + if LINUX: + # https://github.com/giampaolo/psutil/pull/2288 + with pytest.raises(psutil.ZombieProcess) as cm: + proc.cmdline() + self._check_proc_exc(proc, cm.value) + with pytest.raises(psutil.ZombieProcess) as cm: + proc.exe() + self._check_proc_exc(proc, cm.value) + with pytest.raises(psutil.ZombieProcess) as cm: + proc.memory_maps() + self._check_proc_exc(proc, cm.value) + # Zombie cannot be signaled or terminated. + proc.suspend() + proc.resume() + proc.terminate() + proc.kill() + assert proc.is_running() + assert psutil.pid_exists(proc.pid) + assert proc.pid in psutil.pids() + assert proc.pid in [x.pid for x in psutil.process_iter()] + psutil._pmap = {} + assert proc.pid in [x.pid for x in psutil.process_iter()] + + # Its parent should 'see' it (edit: not true on BSD and MACOS). + # descendants = [x.pid for x in psutil.Process().children( + # recursive=True)] + # self.assertIn(proc.pid, descendants) + + # __eq__ can't be relied upon because creation time may not be + # querable. + # self.assertEqual(proc, psutil.Process(proc.pid)) + + # XXX should we also assume ppid() to be usable? Note: this + # would be an important use case as the only way to get + # rid of a zombie is to kill its parent. + # self.assertEqual(proc.ppid(), os.getpid()) + + +@pytest.mark.skipif(PYPY, reason="unreliable on PYPY") +class TestMemoryLeak(PsutilTestCase): + """Test framework class for detecting function memory leaks, + typically functions implemented in C which forgot to free() memory + from the heap. It does so by checking whether the process memory + usage increased before and after calling the function many times. + + Note that this is hard (probably impossible) to do reliably, due + to how the OS handles memory, the GC and so on (memory can even + decrease!). In order to avoid false positives, in case of failure + (mem > 0) we retry the test for up to 5 times, increasing call + repetitions each time. If the memory keeps increasing then it's a + failure. + + If available (Linux, OSX, Windows), USS memory is used for comparison, + since it's supposed to be more precise, see: + https://gmpy.dev/blog/2016/real-process-memory-and-environ-in-python + If not, RSS memory is used. mallinfo() on Linux and _heapwalk() on + Windows may give even more precision, but at the moment are not + implemented. + + PyPy appears to be completely unstable for this framework, probably + because of its JIT, so tests on PYPY are skipped. + + Usage: + + class TestLeaks(psutil.tests.TestMemoryLeak): + + def test_fun(self): + self.execute(some_function) + """ + + # Configurable class attrs. + times = 200 + warmup_times = 10 + tolerance = 0 # memory + retries = 10 if CI_TESTING else 5 + verbose = True + _thisproc = psutil.Process() + _psutil_debug_orig = bool(os.getenv('PSUTIL_DEBUG')) + + @classmethod + def setUpClass(cls): + psutil._set_debug(False) # avoid spamming to stderr + + @classmethod + def tearDownClass(cls): + psutil._set_debug(cls._psutil_debug_orig) + + def _get_mem(self): + # USS is the closest thing we have to "real" memory usage and it + # should be less likely to produce false positives. + mem = self._thisproc.memory_full_info() + return getattr(mem, "uss", mem.rss) + + def _get_num_fds(self): + if POSIX: + return self._thisproc.num_fds() + else: + return self._thisproc.num_handles() + + def _log(self, msg): + if self.verbose: + print_color(msg, color="yellow", file=sys.stderr) + + def _check_fds(self, fun): + """Makes sure num_fds() (POSIX) or num_handles() (Windows) does + not increase after calling a function. Used to discover forgotten + close(2) and CloseHandle syscalls. + """ + before = self._get_num_fds() + self.call(fun) + after = self._get_num_fds() + diff = after - before + if diff < 0: + msg = ( + f"negative diff {diff!r} (gc probably collected a" + " resource from a previous test)" + ) + raise self.fail(msg) + if diff > 0: + type_ = "fd" if POSIX else "handle" + if diff > 1: + type_ += "s" + msg = f"{diff} unclosed {type_} after calling {fun!r}" + raise self.fail(msg) + + def _call_ntimes(self, fun, times): + """Get 2 distinct memory samples, before and after having + called fun repeatedly, and return the memory difference. + """ + gc.collect(generation=1) + mem1 = self._get_mem() + for x in range(times): + ret = self.call(fun) + del x, ret + gc.collect(generation=1) + mem2 = self._get_mem() + assert gc.garbage == [] + diff = mem2 - mem1 # can also be negative + return diff + + def _check_mem(self, fun, times, retries, tolerance): + messages = [] + prev_mem = 0 + increase = times + for idx in range(1, retries + 1): + mem = self._call_ntimes(fun, times) + msg = "Run #{}: extra-mem={}, per-call={}, calls={}".format( + idx, + bytes2human(mem), + bytes2human(mem / times), + times, + ) + messages.append(msg) + success = mem <= tolerance or mem <= prev_mem + if success: + if idx > 1: + self._log(msg) + return + else: + if idx == 1: + print() # noqa: T201 + self._log(msg) + times += increase + prev_mem = mem + raise self.fail(". ".join(messages)) + + # --- + + def call(self, fun): + return fun() + + def execute( + self, fun, times=None, warmup_times=None, retries=None, tolerance=None + ): + """Test a callable.""" + times = times if times is not None else self.times + warmup_times = ( + warmup_times if warmup_times is not None else self.warmup_times + ) + retries = retries if retries is not None else self.retries + tolerance = tolerance if tolerance is not None else self.tolerance + try: + assert times >= 1, "times must be >= 1" + assert warmup_times >= 0, "warmup_times must be >= 0" + assert retries >= 0, "retries must be >= 0" + assert tolerance >= 0, "tolerance must be >= 0" + except AssertionError as err: + raise ValueError(str(err)) + + self._call_ntimes(fun, warmup_times) # warm up + self._check_fds(fun) + self._check_mem(fun, times=times, retries=retries, tolerance=tolerance) + + def execute_w_exc(self, exc, fun, **kwargs): + """Convenience method to test a callable while making sure it + raises an exception on every call. + """ + + def call(): + self.assertRaises(exc, fun) + + self.execute(call, **kwargs) + + +def print_sysinfo(): + import collections + import datetime + import getpass + import locale + import pprint + + try: + import pip + except ImportError: + pip = None + try: + import wheel + except ImportError: + wheel = None + + info = collections.OrderedDict() + + # OS + if psutil.LINUX and shutil.which("lsb_release"): + info['OS'] = sh('lsb_release -d -s') + elif psutil.OSX: + info['OS'] = f"Darwin {platform.mac_ver()[0]}" + elif psutil.WINDOWS: + info['OS'] = "Windows " + ' '.join(map(str, platform.win32_ver())) + if hasattr(platform, 'win32_edition'): + info['OS'] += ", " + platform.win32_edition() + else: + info['OS'] = f"{platform.system()} {platform.version()}" + info['arch'] = ', '.join( + list(platform.architecture()) + [platform.machine()] + ) + if psutil.POSIX: + info['kernel'] = platform.uname()[2] + + # python + info['python'] = ', '.join([ + platform.python_implementation(), + platform.python_version(), + platform.python_compiler(), + ]) + info['pip'] = getattr(pip, '__version__', 'not installed') + if wheel is not None: + info['pip'] += f" (wheel={wheel.__version__})" + + # UNIX + if psutil.POSIX: + if shutil.which("gcc"): + out = sh(['gcc', '--version']) + info['gcc'] = str(out).split('\n')[0] + else: + info['gcc'] = 'not installed' + s = platform.libc_ver()[1] + if s: + info['glibc'] = s + + # system + info['fs-encoding'] = sys.getfilesystemencoding() + lang = locale.getlocale() + info['lang'] = f"{lang[0]}, {lang[1]}" + info['boot-time'] = datetime.datetime.fromtimestamp( + psutil.boot_time() + ).strftime("%Y-%m-%d %H:%M:%S") + info['time'] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + info['user'] = getpass.getuser() + info['home'] = os.path.expanduser("~") + info['cwd'] = os.getcwd() + info['pyexe'] = PYTHON_EXE + info['hostname'] = platform.node() + info['PID'] = os.getpid() + + # metrics + info['cpus'] = psutil.cpu_count() + info['loadavg'] = "{:.1f}%, {:.1f}%, {:.1f}%".format( + *tuple(x / psutil.cpu_count() * 100 for x in psutil.getloadavg()) + ) + mem = psutil.virtual_memory() + info['memory'] = "{}%%, used={}, total={}".format( + int(mem.percent), + bytes2human(mem.used), + bytes2human(mem.total), + ) + swap = psutil.swap_memory() + info['swap'] = "{}%%, used={}, total={}".format( + int(swap.percent), + bytes2human(swap.used), + bytes2human(swap.total), + ) + info['pids'] = len(psutil.pids()) + pinfo = psutil.Process().as_dict() + pinfo.pop('memory_maps', None) + info['proc'] = pprint.pformat(pinfo) + + print("=" * 70, file=sys.stderr) # noqa: T201 + for k, v in info.items(): + print("{:<17} {}".format(k + ":", v), file=sys.stderr) # noqa: T201 + print("=" * 70, file=sys.stderr) # noqa: T201 + sys.stdout.flush() + + # if WINDOWS: + # os.system("tasklist") + # elif shutil.which("ps"): + # os.system("ps aux") + # print("=" * 70, file=sys.stderr) + + sys.stdout.flush() + + +def is_win_secure_system_proc(pid): + # see: https://github.com/giampaolo/psutil/issues/2338 + @memoize + def get_procs(): + ret = {} + out = sh("tasklist.exe /NH /FO csv") + for line in out.splitlines()[1:]: + bits = [x.replace('"', "") for x in line.split(",")] + name, pid = bits[0], int(bits[1]) + ret[pid] = name + return ret + + try: + return get_procs()[pid] == "Secure System" + except KeyError: + return False + + +def _get_eligible_cpu(): + p = psutil.Process() + if hasattr(p, "cpu_num"): + return p.cpu_num() + elif hasattr(p, "cpu_affinity"): + return random.choice(p.cpu_affinity()) + return 0 + + +class process_namespace: + """A container that lists all Process class method names + some + reasonable parameters to be called with. Utility methods (parent(), + children(), ...) are excluded. + + >>> ns = process_namespace(psutil.Process()) + >>> for fun, name in ns.iter(ns.getters): + ... fun() + """ + + utils = [('cpu_percent', (), {}), ('memory_percent', (), {})] + + ignored = [ + ('as_dict', (), {}), + ('children', (), {'recursive': True}), + ('connections', (), {}), # deprecated + ('is_running', (), {}), + ('oneshot', (), {}), + ('parent', (), {}), + ('parents', (), {}), + ('pid', (), {}), + ('wait', (0,), {}), + ] + + getters = [ + ('cmdline', (), {}), + ('cpu_times', (), {}), + ('create_time', (), {}), + ('cwd', (), {}), + ('exe', (), {}), + ('memory_full_info', (), {}), + ('memory_info', (), {}), + ('name', (), {}), + ('net_connections', (), {'kind': 'all'}), + ('nice', (), {}), + ('num_ctx_switches', (), {}), + ('num_threads', (), {}), + ('open_files', (), {}), + ('ppid', (), {}), + ('status', (), {}), + ('threads', (), {}), + ('username', (), {}), + ] + if POSIX: + getters += [('uids', (), {})] + getters += [('gids', (), {})] + getters += [('terminal', (), {})] + getters += [('num_fds', (), {})] + if HAS_PROC_IO_COUNTERS: + getters += [('io_counters', (), {})] + if HAS_IONICE: + getters += [('ionice', (), {})] + if HAS_RLIMIT: + getters += [('rlimit', (psutil.RLIMIT_NOFILE,), {})] + if HAS_CPU_AFFINITY: + getters += [('cpu_affinity', (), {})] + if HAS_PROC_CPU_NUM: + getters += [('cpu_num', (), {})] + if HAS_ENVIRON: + getters += [('environ', (), {})] + if WINDOWS: + getters += [('num_handles', (), {})] + if HAS_MEMORY_MAPS: + getters += [('memory_maps', (), {'grouped': False})] + + setters = [] + if POSIX: + setters += [('nice', (0,), {})] + else: + setters += [('nice', (psutil.NORMAL_PRIORITY_CLASS,), {})] + if HAS_RLIMIT: + setters += [('rlimit', (psutil.RLIMIT_NOFILE, (1024, 4096)), {})] + if HAS_IONICE: + if LINUX: + setters += [('ionice', (psutil.IOPRIO_CLASS_NONE, 0), {})] + else: + setters += [('ionice', (psutil.IOPRIO_NORMAL,), {})] + if HAS_CPU_AFFINITY: + setters += [('cpu_affinity', ([_get_eligible_cpu()],), {})] + + killers = [ + ('send_signal', (signal.SIGTERM,), {}), + ('suspend', (), {}), + ('resume', (), {}), + ('terminate', (), {}), + ('kill', (), {}), + ] + if WINDOWS: + killers += [('send_signal', (signal.CTRL_C_EVENT,), {})] + killers += [('send_signal', (signal.CTRL_BREAK_EVENT,), {})] + + all = utils + getters + setters + killers + + def __init__(self, proc): + self._proc = proc + + def iter(self, ls, clear_cache=True): + """Given a list of tuples yields a set of (fun, fun_name) tuples + in random order. + """ + ls = list(ls) + random.shuffle(ls) + for fun_name, args, kwds in ls: + if clear_cache: + self.clear_cache() + fun = getattr(self._proc, fun_name) + fun = functools.partial(fun, *args, **kwds) + yield (fun, fun_name) + + def clear_cache(self): + """Clear the cache of a Process instance.""" + self._proc._init(self._proc.pid, _ignore_nsp=True) + + @classmethod + def test_class_coverage(cls, test_class, ls): + """Given a TestCase instance and a list of tuples checks that + the class defines the required test method names. + """ + for fun_name, _, _ in ls: + meth_name = 'test_' + fun_name + if not hasattr(test_class, meth_name): + msg = ( + f"{test_class.__class__.__name__!r} class should define a" + f" {meth_name!r} method" + ) + raise AttributeError(msg) + + @classmethod + def test(cls): + this = {x[0] for x in cls.all} + ignored = {x[0] for x in cls.ignored} + klass = {x for x in dir(psutil.Process) if x[0] != '_'} + leftout = (this | ignored) ^ klass + if leftout: + raise ValueError(f"uncovered Process class names: {leftout!r}") + + +class system_namespace: + """A container that lists all the module-level, system-related APIs. + Utilities such as cpu_percent() are excluded. Usage: + + >>> ns = system_namespace + >>> for fun, name in ns.iter(ns.getters): + ... fun() + """ + + getters = [ + ('boot_time', (), {}), + ('cpu_count', (), {'logical': False}), + ('cpu_count', (), {'logical': True}), + ('cpu_stats', (), {}), + ('cpu_times', (), {'percpu': False}), + ('cpu_times', (), {'percpu': True}), + ('disk_io_counters', (), {'perdisk': True}), + ('disk_partitions', (), {'all': True}), + ('disk_usage', (os.getcwd(),), {}), + ('net_connections', (), {'kind': 'all'}), + ('net_if_addrs', (), {}), + ('net_if_stats', (), {}), + ('net_io_counters', (), {'pernic': True}), + ('pid_exists', (os.getpid(),), {}), + ('pids', (), {}), + ('swap_memory', (), {}), + ('users', (), {}), + ('virtual_memory', (), {}), + ] + if HAS_CPU_FREQ: + if MACOS and platform.machine() == 'arm64': # skipped due to #1892 + pass + else: + getters += [('cpu_freq', (), {'percpu': True})] + if HAS_GETLOADAVG: + getters += [('getloadavg', (), {})] + if HAS_SENSORS_TEMPERATURES: + getters += [('sensors_temperatures', (), {})] + if HAS_SENSORS_FANS: + getters += [('sensors_fans', (), {})] + if HAS_SENSORS_BATTERY: + getters += [('sensors_battery', (), {})] + if WINDOWS: + getters += [('win_service_iter', (), {})] + getters += [('win_service_get', ('alg',), {})] + + ignored = [ + ('process_iter', (), {}), + ('wait_procs', ([psutil.Process()],), {}), + ('cpu_percent', (), {}), + ('cpu_times_percent', (), {}), + ] + + all = getters + + @staticmethod + def iter(ls): + """Given a list of tuples yields a set of (fun, fun_name) tuples + in random order. + """ + ls = list(ls) + random.shuffle(ls) + for fun_name, args, kwds in ls: + fun = getattr(psutil, fun_name) + fun = functools.partial(fun, *args, **kwds) + yield (fun, fun_name) + + test_class_coverage = process_namespace.test_class_coverage + + +def retry_on_failure(retries=NO_RETRIES): + """Decorator which runs a test function and retries N times before + actually failing. + """ + + def logfun(exc): + print(f"{exc!r}, retrying", file=sys.stderr) # noqa: T201 + + return retry( + exception=AssertionError, timeout=None, retries=retries, logfun=logfun + ) + + +def skip_on_access_denied(only_if=None): + """Decorator to Ignore AccessDenied exceptions.""" + + def decorator(fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except psutil.AccessDenied: + if only_if is not None: + if not only_if: + raise + raise pytest.skip("raises AccessDenied") + + return wrapper + + return decorator + + +def skip_on_not_implemented(only_if=None): + """Decorator to Ignore NotImplementedError exceptions.""" + + def decorator(fun): + @functools.wraps(fun) + def wrapper(*args, **kwargs): + try: + return fun(*args, **kwargs) + except NotImplementedError: + if only_if is not None: + if not only_if: + raise + msg = ( + f"{fun.__name__!r} was skipped because it raised" + " NotImplementedError" + ) + raise pytest.skip(msg) + + return wrapper + + return decorator + + +# =================================================================== +# --- network +# =================================================================== + + +# XXX: no longer used +def get_free_port(host='127.0.0.1'): + """Return an unused TCP port. Subject to race conditions.""" + with socket.socket() as sock: + sock.bind((host, 0)) + return sock.getsockname()[1] + + +def bind_socket(family=AF_INET, type=SOCK_STREAM, addr=None): + """Binds a generic socket.""" + if addr is None and family in {AF_INET, AF_INET6}: + addr = ("", 0) + sock = socket.socket(family, type) + try: + if os.name not in {'nt', 'cygwin'}: + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.bind(addr) + if type == socket.SOCK_STREAM: + sock.listen(5) + return sock + except Exception: + sock.close() + raise + + +def bind_unix_socket(name, type=socket.SOCK_STREAM): + """Bind a UNIX socket.""" + assert psutil.POSIX + assert not os.path.exists(name), name + sock = socket.socket(socket.AF_UNIX, type) + try: + sock.bind(name) + if type == socket.SOCK_STREAM: + sock.listen(5) + except Exception: + sock.close() + raise + return sock + + +def tcp_socketpair(family, addr=("", 0)): + """Build a pair of TCP sockets connected to each other. + Return a (server, client) tuple. + """ + with socket.socket(family, SOCK_STREAM) as ll: + ll.bind(addr) + ll.listen(5) + addr = ll.getsockname() + c = socket.socket(family, SOCK_STREAM) + try: + c.connect(addr) + caddr = c.getsockname() + while True: + a, addr = ll.accept() + # check that we've got the correct client + if addr == caddr: + return (a, c) + a.close() + except OSError: + c.close() + raise + + +def unix_socketpair(name): + """Build a pair of UNIX sockets connected to each other through + the same UNIX file name. + Return a (server, client) tuple. + """ + assert psutil.POSIX + server = client = None + try: + server = bind_unix_socket(name, type=socket.SOCK_STREAM) + server.setblocking(0) + client = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + client.setblocking(0) + client.connect(name) + # new = server.accept() + except Exception: + if server is not None: + server.close() + if client is not None: + client.close() + raise + return (server, client) + + +@contextlib.contextmanager +def create_sockets(): + """Open as many socket families / types as possible.""" + socks = [] + fname1 = fname2 = None + try: + socks.extend(( + bind_socket(socket.AF_INET, socket.SOCK_STREAM), + bind_socket(socket.AF_INET, socket.SOCK_DGRAM), + )) + if supports_ipv6(): + socks.extend(( + bind_socket(socket.AF_INET6, socket.SOCK_STREAM), + bind_socket(socket.AF_INET6, socket.SOCK_DGRAM), + )) + if POSIX and HAS_NET_CONNECTIONS_UNIX: + fname1 = get_testfn() + fname2 = get_testfn() + s1, s2 = unix_socketpair(fname1) + s3 = bind_unix_socket(fname2, type=socket.SOCK_DGRAM) + for s in (s1, s2, s3): + socks.append(s) + yield socks + finally: + for s in socks: + s.close() + for fname in (fname1, fname2): + if fname is not None: + safe_rmpath(fname) + + +def check_net_address(addr, family): + """Check a net address validity. Supported families are IPv4, + IPv6 and MAC addresses. + """ + assert isinstance(family, enum.IntEnum), family + if family == socket.AF_INET: + octs = [int(x) for x in addr.split('.')] + assert len(octs) == 4, addr + for num in octs: + assert 0 <= num <= 255, addr + ipaddress.IPv4Address(addr) + elif family == socket.AF_INET6: + assert isinstance(addr, str), addr + ipaddress.IPv6Address(addr) + elif family == psutil.AF_LINK: + assert re.match(r'([a-fA-F0-9]{2}[:|\-]?){6}', addr) is not None, addr + else: + raise ValueError(f"unknown family {family!r}") + + +def check_connection_ntuple(conn): + """Check validity of a connection namedtuple.""" + + def check_ntuple(conn): + has_pid = len(conn) == 7 + assert len(conn) in {6, 7}, len(conn) + assert conn[0] == conn.fd, conn.fd + assert conn[1] == conn.family, conn.family + assert conn[2] == conn.type, conn.type + assert conn[3] == conn.laddr, conn.laddr + assert conn[4] == conn.raddr, conn.raddr + assert conn[5] == conn.status, conn.status + if has_pid: + assert conn[6] == conn.pid, conn.pid + + def check_family(conn): + assert conn.family in {AF_INET, AF_INET6, AF_UNIX}, conn.family + assert isinstance(conn.family, enum.IntEnum), conn + if conn.family == AF_INET: + # actually try to bind the local socket; ignore IPv6 + # sockets as their address might be represented as + # an IPv4-mapped-address (e.g. "::127.0.0.1") + # and that's rejected by bind() + with socket.socket(conn.family, conn.type) as s: + try: + s.bind((conn.laddr[0], 0)) + except OSError as err: + if err.errno != errno.EADDRNOTAVAIL: + raise + elif conn.family == AF_UNIX: + assert conn.status == psutil.CONN_NONE, conn.status + + def check_type(conn): + # SOCK_SEQPACKET may happen in case of AF_UNIX socks + SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) + assert conn.type in { + socket.SOCK_STREAM, + socket.SOCK_DGRAM, + SOCK_SEQPACKET, + }, conn.type + assert isinstance(conn.type, enum.IntEnum), conn + if conn.type == socket.SOCK_DGRAM: + assert conn.status == psutil.CONN_NONE, conn.status + + def check_addrs(conn): + # check IP address and port sanity + for addr in (conn.laddr, conn.raddr): + if conn.family in {AF_INET, AF_INET6}: + assert isinstance(addr, tuple), type(addr) + if not addr: + continue + assert isinstance(addr.port, int), type(addr.port) + assert 0 <= addr.port <= 65535, addr.port + check_net_address(addr.ip, conn.family) + elif conn.family == AF_UNIX: + assert isinstance(addr, str), type(addr) + + def check_status(conn): + assert isinstance(conn.status, str), conn.status + valids = [ + getattr(psutil, x) for x in dir(psutil) if x.startswith('CONN_') + ] + assert conn.status in valids, conn.status + if conn.family in {AF_INET, AF_INET6} and conn.type == SOCK_STREAM: + assert conn.status != psutil.CONN_NONE, conn.status + else: + assert conn.status == psutil.CONN_NONE, conn.status + + check_ntuple(conn) + check_family(conn) + check_type(conn) + check_addrs(conn) + check_status(conn) + + +def filter_proc_net_connections(cons): + """Our process may start with some open UNIX sockets which are not + initialized by us, invalidating unit tests. + """ + new = [] + for conn in cons: + if POSIX and conn.family == socket.AF_UNIX: + if MACOS and "/syslog" in conn.raddr: + debug(f"skipping {conn}") + continue + new.append(conn) + return new + + +# =================================================================== +# --- import utils +# =================================================================== + + +def reload_module(module): + return importlib.reload(module) + + +def import_module_by_path(path): + name = os.path.splitext(os.path.basename(path))[0] + spec = importlib.util.spec_from_file_location(name, path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# =================================================================== +# --- others +# =================================================================== + + +def warn(msg): + """Raise a warning msg.""" + warnings.warn(msg, UserWarning, stacklevel=2) + + +def is_namedtuple(x): + """Check if object is an instance of namedtuple.""" + t = type(x) + b = t.__bases__ + if len(b) != 1 or b[0] is not tuple: + return False + f = getattr(t, '_fields', None) + if not isinstance(f, tuple): + return False + return all(isinstance(n, str) for n in f) + + +if POSIX: + + @contextlib.contextmanager + def copyload_shared_lib(suffix=""): + """Ctx manager which picks up a random shared CO lib used + by this process, copies it in another location and loads it + in memory via ctypes. Return the new absolutized path. + """ + exe = 'pypy' if PYPY else 'python' + ext = ".so" + dst = get_testfn(suffix=suffix + ext) + libs = [ + x.path + for x in psutil.Process().memory_maps() + if os.path.splitext(x.path)[1] == ext and exe in x.path.lower() + ] + src = random.choice(libs) + shutil.copyfile(src, dst) + try: + ctypes.CDLL(dst) + yield dst + finally: + safe_rmpath(dst) + +else: + + @contextlib.contextmanager + def copyload_shared_lib(suffix=""): + """Ctx manager which picks up a random shared DLL lib used + by this process, copies it in another location and loads it + in memory via ctypes. + Return the new absolutized, normcased path. + """ + from ctypes import WinError + from ctypes import wintypes + + ext = ".dll" + dst = get_testfn(suffix=suffix + ext) + libs = [ + x.path + for x in psutil.Process().memory_maps() + if x.path.lower().endswith(ext) + and 'python' in os.path.basename(x.path).lower() + and 'wow64' not in x.path.lower() + ] + if PYPY and not libs: + libs = [ + x.path + for x in psutil.Process().memory_maps() + if 'pypy' in os.path.basename(x.path).lower() + ] + src = random.choice(libs) + shutil.copyfile(src, dst) + cfile = None + try: + cfile = ctypes.WinDLL(dst) + yield dst + finally: + # Work around OverflowError: + # - https://ci.appveyor.com/project/giampaolo/psutil/build/1207/ + # job/o53330pbnri9bcw7 + # - http://bugs.python.org/issue30286 + # - http://stackoverflow.com/questions/23522055 + if cfile is not None: + FreeLibrary = ctypes.windll.kernel32.FreeLibrary + FreeLibrary.argtypes = [wintypes.HMODULE] + ret = FreeLibrary(cfile._handle) + if ret == 0: + raise WinError() + safe_rmpath(dst) + + +# =================================================================== +# --- Exit funs (first is executed last) +# =================================================================== + + +# this is executed first +@atexit.register +def cleanup_test_procs(): + reap_children(recursive=True) + + +# atexit module does not execute exit functions in case of SIGTERM, which +# gets sent to test subprocesses, which is a problem if they import this +# module. With this it will. See: +# https://gmpy.dev/blog/2016/how-to-always-execute-exit-functions-in-python +if POSIX: + signal.signal(signal.SIGTERM, lambda sig, _: sys.exit(sig)) diff --git a/lib/python3.12/site-packages/psutil/tests/__main__.py b/lib/python3.12/site-packages/psutil/tests/__main__.py new file mode 100644 index 0000000000000000000000000000000000000000..ce6fc24c7f09a273983c81d9ae5b2180e83fbb9f --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/__main__.py @@ -0,0 +1,12 @@ +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Run unit tests. This is invoked by: +$ python -m psutil.tests. +""" + +from psutil.tests import pytest + + +pytest.main(["-v", "-s", "--tb=short"]) diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0b4758ff6d53dda00a49e1f480c41ef07f2d8aeb Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/__main__.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/__main__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..27c7f0e06ea74ff6572f5943ac498b6c70acaf46 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/__main__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_aix.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_aix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d489f5615de917a97673c91a47514af2ce91cd6c Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_aix.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_bsd.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_bsd.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5de0fa5511cb1c52a1863b1ef499da37503995d3 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_bsd.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_connections.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_connections.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b713461d04699903aa7d7806025d0b05922e9b15 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_connections.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_contracts.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_contracts.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..df43f2a6aefca937b608b56872f00d3245089bf0 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_contracts.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_memleaks.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_memleaks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e7ae9b8b054c8128fb4bb89eff4878c4ac3cf735 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_memleaks.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_misc.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_misc.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3e2bb90160a6ab7283c2b286cf05056460a9ed81 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_misc.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_osx.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_osx.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8193d422e544c865a59672792ff24928ec964691 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_osx.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_posix.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_posix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..520527dff73aa21142240159a0bd05e4afd462ae Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_posix.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_process.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_process.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..93d52d3c2683e023d776ef32bcfd689cf887b01f Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_process.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_process_all.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_process_all.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e68fef67236146665bb6824e3922880f68379566 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_process_all.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_scripts.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_scripts.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ec2a05e54e8e23dcadb9326cc389e24944ceb13 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_scripts.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_sunos.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_sunos.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e09f6e7d3d4d7f8711aa67cb71378d49db42e507 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_sunos.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_system.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_system.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..01d36fbf541a07999910e923c5b89c30e6404bb9 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_system.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_testutils.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_testutils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..55e35e6f0625a3e795dc460235f735e6d13ae7ec Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_testutils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_unicode.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_unicode.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..53f3f2d5af802eb822a8f15bcf5710873f07d1aa Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_unicode.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/__pycache__/test_windows.cpython-312.pyc b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_windows.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..820e7746585367028ac004e026501ceb03381784 Binary files /dev/null and b/lib/python3.12/site-packages/psutil/tests/__pycache__/test_windows.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/psutil/tests/test_aix.py b/lib/python3.12/site-packages/psutil/tests/test_aix.py new file mode 100644 index 0000000000000000000000000000000000000000..10934c12d41ee19b641fca103bee992b9336fc16 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_aix.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola' +# Copyright (c) 2017, Arnon Yaari +# All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""AIX specific tests.""" + +import re + +import psutil +from psutil import AIX +from psutil.tests import PsutilTestCase +from psutil.tests import pytest +from psutil.tests import sh + + +@pytest.mark.skipif(not AIX, reason="AIX only") +class AIXSpecificTestCase(PsutilTestCase): + def test_virtual_memory(self): + out = sh('/usr/bin/svmon -O unit=KB') + re_pattern = r"memory\s*" + for field in [ + "size", + "inuse", + "free", + "pin", + "virtual", + "available", + "mmode", + ]: + re_pattern += rf"(?P<{field}>\S+)\s+" + matchobj = re.search(re_pattern, out) + + assert matchobj is not None + + KB = 1024 + total = int(matchobj.group("size")) * KB + available = int(matchobj.group("available")) * KB + used = int(matchobj.group("inuse")) * KB + free = int(matchobj.group("free")) * KB + + psutil_result = psutil.virtual_memory() + + # TOLERANCE_SYS_MEM from psutil.tests is not enough. For some reason + # we're seeing differences of ~1.2 MB. 2 MB is still a good tolerance + # when compared to GBs. + TOLERANCE_SYS_MEM = 2 * KB * KB # 2 MB + assert psutil_result.total == total + assert abs(psutil_result.used - used) < TOLERANCE_SYS_MEM + assert abs(psutil_result.available - available) < TOLERANCE_SYS_MEM + assert abs(psutil_result.free - free) < TOLERANCE_SYS_MEM + + def test_swap_memory(self): + out = sh('/usr/sbin/lsps -a') + # From the man page, "The size is given in megabytes" so we assume + # we'll always have 'MB' in the result + # TODO maybe try to use "swap -l" to check "used" too, but its units + # are not guaranteed to be "MB" so parsing may not be consistent + matchobj = re.search( + r"(?P\S+)\s+" + r"(?P\S+)\s+" + r"(?P\S+)\s+" + r"(?P\d+)MB", + out, + ) + + assert matchobj is not None + + total_mb = int(matchobj.group("size")) + MB = 1024**2 + psutil_result = psutil.swap_memory() + # we divide our result by MB instead of multiplying the lsps value by + # MB because lsps may round down, so we round down too + assert int(psutil_result.total / MB) == total_mb + + def test_cpu_stats(self): + out = sh('/usr/bin/mpstat -a') + + re_pattern = r"ALL\s*" + for field in [ + "min", + "maj", + "mpcs", + "mpcr", + "dev", + "soft", + "dec", + "ph", + "cs", + "ics", + "bound", + "rq", + "push", + "S3pull", + "S3grd", + "S0rd", + "S1rd", + "S2rd", + "S3rd", + "S4rd", + "S5rd", + "sysc", + ]: + re_pattern += rf"(?P<{field}>\S+)\s+" + matchobj = re.search(re_pattern, out) + + assert matchobj is not None + + # numbers are usually in the millions so 1000 is ok for tolerance + CPU_STATS_TOLERANCE = 1000 + psutil_result = psutil.cpu_stats() + assert ( + abs(psutil_result.ctx_switches - int(matchobj.group("cs"))) + < CPU_STATS_TOLERANCE + ) + assert ( + abs(psutil_result.syscalls - int(matchobj.group("sysc"))) + < CPU_STATS_TOLERANCE + ) + assert ( + abs(psutil_result.interrupts - int(matchobj.group("dev"))) + < CPU_STATS_TOLERANCE + ) + assert ( + abs(psutil_result.soft_interrupts - int(matchobj.group("soft"))) + < CPU_STATS_TOLERANCE + ) + + def test_cpu_count_logical(self): + out = sh('/usr/bin/mpstat -a') + mpstat_lcpu = int(re.search(r"lcpu=(\d+)", out).group(1)) + psutil_lcpu = psutil.cpu_count(logical=True) + assert mpstat_lcpu == psutil_lcpu + + def test_net_if_addrs_names(self): + out = sh('/etc/ifconfig -l') + ifconfig_names = set(out.split()) + psutil_names = set(psutil.net_if_addrs().keys()) + assert ifconfig_names == psutil_names diff --git a/lib/python3.12/site-packages/psutil/tests/test_bsd.py b/lib/python3.12/site-packages/psutil/tests/test_bsd.py new file mode 100644 index 0000000000000000000000000000000000000000..2786c348577855207cc4169714721bebb8776828 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_bsd.py @@ -0,0 +1,593 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# TODO: (FreeBSD) add test for comparing connections with 'sockstat' cmd. + + +"""Tests specific to all BSD platforms.""" + +import datetime +import os +import re +import shutil +import time + +import psutil +from psutil import BSD +from psutil import FREEBSD +from psutil import NETBSD +from psutil import OPENBSD +from psutil.tests import HAS_BATTERY +from psutil.tests import TOLERANCE_SYS_MEM +from psutil.tests import PsutilTestCase +from psutil.tests import pytest +from psutil.tests import retry_on_failure +from psutil.tests import sh +from psutil.tests import spawn_testproc +from psutil.tests import terminate + + +if BSD: + from psutil._psutil_posix import getpagesize + + PAGESIZE = getpagesize() + # muse requires root privileges + MUSE_AVAILABLE = os.getuid() == 0 and shutil.which("muse") +else: + PAGESIZE = None + MUSE_AVAILABLE = False + + +def sysctl(cmdline): + """Expects a sysctl command with an argument and parse the result + returning only the value of interest. + """ + result = sh("sysctl " + cmdline) + if FREEBSD: + result = result[result.find(": ") + 2 :] + elif OPENBSD or NETBSD: + result = result[result.find("=") + 1 :] + try: + return int(result) + except ValueError: + return result + + +def muse(field): + """Thin wrapper around 'muse' cmdline utility.""" + out = sh('muse') + for line in out.split('\n'): + if line.startswith(field): + break + else: + raise ValueError("line not found") + return int(line.split()[1]) + + +# ===================================================================== +# --- All BSD* +# ===================================================================== + + +@pytest.mark.skipif(not BSD, reason="BSD only") +class BSDTestCase(PsutilTestCase): + """Generic tests common to all BSD variants.""" + + @classmethod + def setUpClass(cls): + cls.pid = spawn_testproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + @pytest.mark.skipif(NETBSD, reason="-o lstart doesn't work on NETBSD") + def test_process_create_time(self): + output = sh(f"ps -o lstart -p {self.pid}") + start_ps = output.replace('STARTED', '').strip() + start_psutil = psutil.Process(self.pid).create_time() + start_psutil = time.strftime( + "%a %b %e %H:%M:%S %Y", time.localtime(start_psutil) + ) + assert start_ps == start_psutil + + def test_disks(self): + # test psutil.disk_usage() and psutil.disk_partitions() + # against "df -a" + def df(path): + out = sh(f'df -k "{path}"').strip() + lines = out.split('\n') + lines.pop(0) + line = lines.pop(0) + dev, total, used, free = line.split()[:4] + if dev == 'none': + dev = '' + total = int(total) * 1024 + used = int(used) * 1024 + free = int(free) * 1024 + return dev, total, used, free + + for part in psutil.disk_partitions(all=False): + usage = psutil.disk_usage(part.mountpoint) + dev, total, used, free = df(part.mountpoint) + assert part.device == dev + assert usage.total == total + # 10 MB tolerance + if abs(usage.free - free) > 10 * 1024 * 1024: + raise self.fail(f"psutil={usage.free}, df={free}") + if abs(usage.used - used) > 10 * 1024 * 1024: + raise self.fail(f"psutil={usage.used}, df={used}") + + @pytest.mark.skipif( + not shutil.which("sysctl"), reason="sysctl cmd not available" + ) + def test_cpu_count_logical(self): + syst = sysctl("hw.ncpu") + assert psutil.cpu_count(logical=True) == syst + + @pytest.mark.skipif( + not shutil.which("sysctl"), reason="sysctl cmd not available" + ) + @pytest.mark.skipif( + NETBSD, reason="skipped on NETBSD" # we check /proc/meminfo + ) + def test_virtual_memory_total(self): + num = sysctl('hw.physmem') + assert num == psutil.virtual_memory().total + + @pytest.mark.skipif( + not shutil.which("ifconfig"), reason="ifconfig cmd not available" + ) + def test_net_if_stats(self): + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + assert stats.isup == ('RUNNING' in out) + if "mtu" in out: + assert stats.mtu == int(re.findall(r'mtu (\d+)', out)[0]) + + +# ===================================================================== +# --- FreeBSD +# ===================================================================== + + +@pytest.mark.skipif(not FREEBSD, reason="FREEBSD only") +class FreeBSDPsutilTestCase(PsutilTestCase): + @classmethod + def setUpClass(cls): + cls.pid = spawn_testproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + @retry_on_failure() + def test_memory_maps(self): + out = sh(f"procstat -v {self.pid}") + maps = psutil.Process(self.pid).memory_maps(grouped=False) + lines = out.split('\n')[1:] + while lines: + line = lines.pop() + fields = line.split() + _, start, stop, _perms, res = fields[:5] + map = maps.pop() + assert f"{start}-{stop}" == map.addr + assert int(res) == map.rss + if not map.path.startswith('['): + assert fields[10] == map.path + + def test_exe(self): + out = sh(f"procstat -b {self.pid}") + assert psutil.Process(self.pid).exe() == out.split('\n')[1].split()[-1] + + def test_cmdline(self): + out = sh(f"procstat -c {self.pid}") + assert ' '.join(psutil.Process(self.pid).cmdline()) == ' '.join( + out.split('\n')[1].split()[2:] + ) + + def test_uids_gids(self): + out = sh(f"procstat -s {self.pid}") + euid, ruid, suid, egid, rgid, sgid = out.split('\n')[1].split()[2:8] + p = psutil.Process(self.pid) + uids = p.uids() + gids = p.gids() + assert uids.real == int(ruid) + assert uids.effective == int(euid) + assert uids.saved == int(suid) + assert gids.real == int(rgid) + assert gids.effective == int(egid) + assert gids.saved == int(sgid) + + @retry_on_failure() + def test_ctx_switches(self): + tested = [] + out = sh(f"procstat -r {self.pid}") + p = psutil.Process(self.pid) + for line in out.split('\n'): + line = line.lower().strip() + if ' voluntary context' in line: + pstat_value = int(line.split()[-1]) + psutil_value = p.num_ctx_switches().voluntary + assert pstat_value == psutil_value + tested.append(None) + elif ' involuntary context' in line: + pstat_value = int(line.split()[-1]) + psutil_value = p.num_ctx_switches().involuntary + assert pstat_value == psutil_value + tested.append(None) + if len(tested) != 2: + raise RuntimeError("couldn't find lines match in procstat out") + + @retry_on_failure() + def test_cpu_times(self): + tested = [] + out = sh(f"procstat -r {self.pid}") + p = psutil.Process(self.pid) + for line in out.split('\n'): + line = line.lower().strip() + if 'user time' in line: + pstat_value = float('0.' + line.split()[-1].split('.')[-1]) + psutil_value = p.cpu_times().user + assert pstat_value == psutil_value + tested.append(None) + elif 'system time' in line: + pstat_value = float('0.' + line.split()[-1].split('.')[-1]) + psutil_value = p.cpu_times().system + assert pstat_value == psutil_value + tested.append(None) + if len(tested) != 2: + raise RuntimeError("couldn't find lines match in procstat out") + + +@pytest.mark.skipif(not FREEBSD, reason="FREEBSD only") +class FreeBSDSystemTestCase(PsutilTestCase): + @staticmethod + def parse_swapinfo(): + # the last line is always the total + output = sh("swapinfo -k").splitlines()[-1] + parts = re.split(r'\s+', output) + + if not parts: + raise ValueError(f"Can't parse swapinfo: {output}") + + # the size is in 1k units, so multiply by 1024 + total, used, free = (int(p) * 1024 for p in parts[1:4]) + return total, used, free + + def test_cpu_frequency_against_sysctl(self): + # Currently only cpu 0 is frequency is supported in FreeBSD + # All other cores use the same frequency. + sensor = "dev.cpu.0.freq" + try: + sysctl_result = int(sysctl(sensor)) + except RuntimeError: + raise pytest.skip("frequencies not supported by kernel") + assert psutil.cpu_freq().current == sysctl_result + + sensor = "dev.cpu.0.freq_levels" + sysctl_result = sysctl(sensor) + # sysctl returns a string of the format: + # / /... + # Ordered highest available to lowest available. + max_freq = int(sysctl_result.split()[0].split("/")[0]) + min_freq = int(sysctl_result.split()[-1].split("/")[0]) + assert psutil.cpu_freq().max == max_freq + assert psutil.cpu_freq().min == min_freq + + # --- virtual_memory(); tests against sysctl + + @retry_on_failure() + def test_vmem_active(self): + syst = sysctl("vm.stats.vm.v_active_count") * PAGESIZE + assert abs(psutil.virtual_memory().active - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_inactive(self): + syst = sysctl("vm.stats.vm.v_inactive_count") * PAGESIZE + assert abs(psutil.virtual_memory().inactive - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_wired(self): + syst = sysctl("vm.stats.vm.v_wire_count") * PAGESIZE + assert abs(psutil.virtual_memory().wired - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_cached(self): + syst = sysctl("vm.stats.vm.v_cache_count") * PAGESIZE + assert abs(psutil.virtual_memory().cached - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_free(self): + syst = sysctl("vm.stats.vm.v_free_count") * PAGESIZE + assert abs(psutil.virtual_memory().free - syst) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_buffers(self): + syst = sysctl("vfs.bufspace") + assert abs(psutil.virtual_memory().buffers - syst) < TOLERANCE_SYS_MEM + + # --- virtual_memory(); tests against muse + + @pytest.mark.skipif(not MUSE_AVAILABLE, reason="muse not installed") + def test_muse_vmem_total(self): + num = muse('Total') + assert psutil.virtual_memory().total == num + + @pytest.mark.skipif(not MUSE_AVAILABLE, reason="muse not installed") + @retry_on_failure() + def test_muse_vmem_active(self): + num = muse('Active') + assert abs(psutil.virtual_memory().active - num) < TOLERANCE_SYS_MEM + + @pytest.mark.skipif(not MUSE_AVAILABLE, reason="muse not installed") + @retry_on_failure() + def test_muse_vmem_inactive(self): + num = muse('Inactive') + assert abs(psutil.virtual_memory().inactive - num) < TOLERANCE_SYS_MEM + + @pytest.mark.skipif(not MUSE_AVAILABLE, reason="muse not installed") + @retry_on_failure() + def test_muse_vmem_wired(self): + num = muse('Wired') + assert abs(psutil.virtual_memory().wired - num) < TOLERANCE_SYS_MEM + + @pytest.mark.skipif(not MUSE_AVAILABLE, reason="muse not installed") + @retry_on_failure() + def test_muse_vmem_cached(self): + num = muse('Cache') + assert abs(psutil.virtual_memory().cached - num) < TOLERANCE_SYS_MEM + + @pytest.mark.skipif(not MUSE_AVAILABLE, reason="muse not installed") + @retry_on_failure() + def test_muse_vmem_free(self): + num = muse('Free') + assert abs(psutil.virtual_memory().free - num) < TOLERANCE_SYS_MEM + + @pytest.mark.skipif(not MUSE_AVAILABLE, reason="muse not installed") + @retry_on_failure() + def test_muse_vmem_buffers(self): + num = muse('Buffer') + assert abs(psutil.virtual_memory().buffers - num) < TOLERANCE_SYS_MEM + + def test_cpu_stats_ctx_switches(self): + assert ( + abs( + psutil.cpu_stats().ctx_switches + - sysctl('vm.stats.sys.v_swtch') + ) + < 1000 + ) + + def test_cpu_stats_interrupts(self): + assert ( + abs(psutil.cpu_stats().interrupts - sysctl('vm.stats.sys.v_intr')) + < 1000 + ) + + def test_cpu_stats_soft_interrupts(self): + assert ( + abs( + psutil.cpu_stats().soft_interrupts + - sysctl('vm.stats.sys.v_soft') + ) + < 1000 + ) + + @retry_on_failure() + def test_cpu_stats_syscalls(self): + # pretty high tolerance but it looks like it's OK. + assert ( + abs(psutil.cpu_stats().syscalls - sysctl('vm.stats.sys.v_syscall')) + < 200000 + ) + + # def test_cpu_stats_traps(self): + # self.assertAlmostEqual(psutil.cpu_stats().traps, + # sysctl('vm.stats.sys.v_trap'), delta=1000) + + # --- swap memory + + def test_swapmem_free(self): + _total, _used, free = self.parse_swapinfo() + assert abs(psutil.swap_memory().free - free) < TOLERANCE_SYS_MEM + + def test_swapmem_used(self): + _total, used, _free = self.parse_swapinfo() + assert abs(psutil.swap_memory().used - used) < TOLERANCE_SYS_MEM + + def test_swapmem_total(self): + total, _used, _free = self.parse_swapinfo() + assert abs(psutil.swap_memory().total - total) < TOLERANCE_SYS_MEM + + # --- others + + def test_boot_time(self): + s = sysctl('sysctl kern.boottime') + s = s[s.find(" sec = ") + 7 :] + s = s[: s.find(',')] + btime = int(s) + assert btime == psutil.boot_time() + + # --- sensors_battery + + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_sensors_battery(self): + def secs2hours(secs): + m, _s = divmod(secs, 60) + h, m = divmod(m, 60) + return f"{int(h)}:{int(m):02}" + + out = sh("acpiconf -i 0") + fields = {x.split('\t')[0]: x.split('\t')[-1] for x in out.split("\n")} + metrics = psutil.sensors_battery() + percent = int(fields['Remaining capacity:'].replace('%', '')) + remaining_time = fields['Remaining time:'] + assert metrics.percent == percent + if remaining_time == 'unknown': + assert metrics.secsleft == psutil.POWER_TIME_UNLIMITED + else: + assert secs2hours(metrics.secsleft) == remaining_time + + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_sensors_battery_against_sysctl(self): + assert psutil.sensors_battery().percent == sysctl( + "hw.acpi.battery.life" + ) + assert psutil.sensors_battery().power_plugged == ( + sysctl("hw.acpi.acline") == 1 + ) + secsleft = psutil.sensors_battery().secsleft + if secsleft < 0: + assert sysctl("hw.acpi.battery.time") == -1 + else: + assert secsleft == sysctl("hw.acpi.battery.time") * 60 + + @pytest.mark.skipif(HAS_BATTERY, reason="has battery") + def test_sensors_battery_no_battery(self): + # If no battery is present one of these calls is supposed + # to fail, see: + # https://github.com/giampaolo/psutil/issues/1074 + with pytest.raises(RuntimeError): + sysctl("hw.acpi.battery.life") + sysctl("hw.acpi.battery.time") + sysctl("hw.acpi.acline") + assert psutil.sensors_battery() is None + + # --- sensors_temperatures + + def test_sensors_temperatures_against_sysctl(self): + num_cpus = psutil.cpu_count(True) + for cpu in range(num_cpus): + sensor = f"dev.cpu.{cpu}.temperature" + # sysctl returns a string in the format 46.0C + try: + sysctl_result = int(float(sysctl(sensor)[:-1])) + except RuntimeError: + raise pytest.skip("temperatures not supported by kernel") + assert ( + abs( + psutil.sensors_temperatures()["coretemp"][cpu].current + - sysctl_result + ) + < 10 + ) + + sensor = f"dev.cpu.{cpu}.coretemp.tjmax" + sysctl_result = int(float(sysctl(sensor)[:-1])) + assert ( + psutil.sensors_temperatures()["coretemp"][cpu].high + == sysctl_result + ) + + +# ===================================================================== +# --- OpenBSD +# ===================================================================== + + +@pytest.mark.skipif(not OPENBSD, reason="OPENBSD only") +class OpenBSDTestCase(PsutilTestCase): + def test_boot_time(self): + s = sysctl('kern.boottime') + sys_bt = datetime.datetime.strptime(s, "%a %b %d %H:%M:%S %Y") + psutil_bt = datetime.datetime.fromtimestamp(psutil.boot_time()) + assert sys_bt == psutil_bt + + +# ===================================================================== +# --- NetBSD +# ===================================================================== + + +@pytest.mark.skipif(not NETBSD, reason="NETBSD only") +class NetBSDTestCase(PsutilTestCase): + @staticmethod + def parse_meminfo(look_for): + with open('/proc/meminfo') as f: + for line in f: + if line.startswith(look_for): + return int(line.split()[1]) * 1024 + raise ValueError(f"can't find {look_for}") + + # --- virtual mem + + def test_vmem_total(self): + assert psutil.virtual_memory().total == self.parse_meminfo("MemTotal:") + + def test_vmem_free(self): + assert ( + abs(psutil.virtual_memory().free - self.parse_meminfo("MemFree:")) + < TOLERANCE_SYS_MEM + ) + + def test_vmem_buffers(self): + assert ( + abs( + psutil.virtual_memory().buffers + - self.parse_meminfo("Buffers:") + ) + < TOLERANCE_SYS_MEM + ) + + def test_vmem_shared(self): + assert ( + abs( + psutil.virtual_memory().shared + - self.parse_meminfo("MemShared:") + ) + < TOLERANCE_SYS_MEM + ) + + def test_vmem_cached(self): + assert ( + abs(psutil.virtual_memory().cached - self.parse_meminfo("Cached:")) + < TOLERANCE_SYS_MEM + ) + + # --- swap mem + + def test_swapmem_total(self): + assert ( + abs(psutil.swap_memory().total - self.parse_meminfo("SwapTotal:")) + < TOLERANCE_SYS_MEM + ) + + def test_swapmem_free(self): + assert ( + abs(psutil.swap_memory().free - self.parse_meminfo("SwapFree:")) + < TOLERANCE_SYS_MEM + ) + + def test_swapmem_used(self): + smem = psutil.swap_memory() + assert smem.used == smem.total - smem.free + + # --- others + + def test_cpu_stats_interrupts(self): + with open('/proc/stat', 'rb') as f: + for line in f: + if line.startswith(b'intr'): + interrupts = int(line.split()[1]) + break + else: + raise ValueError("couldn't find line") + assert abs(psutil.cpu_stats().interrupts - interrupts) < 1000 + + def test_cpu_stats_ctx_switches(self): + with open('/proc/stat', 'rb') as f: + for line in f: + if line.startswith(b'ctxt'): + ctx_switches = int(line.split()[1]) + break + else: + raise ValueError("couldn't find line") + assert abs(psutil.cpu_stats().ctx_switches - ctx_switches) < 1000 diff --git a/lib/python3.12/site-packages/psutil/tests/test_connections.py b/lib/python3.12/site-packages/psutil/tests/test_connections.py new file mode 100644 index 0000000000000000000000000000000000000000..5ddeb855f2a78ba21132c2694b6f426f548cbb48 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_connections.py @@ -0,0 +1,566 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for psutil.net_connections() and Process.net_connections() APIs.""" + +import os +import socket +import textwrap +from contextlib import closing +from socket import AF_INET +from socket import AF_INET6 +from socket import SOCK_DGRAM +from socket import SOCK_STREAM + +import psutil +from psutil import FREEBSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil._common import supports_ipv6 +from psutil.tests import AF_UNIX +from psutil.tests import HAS_NET_CONNECTIONS_UNIX +from psutil.tests import SKIP_SYSCONS +from psutil.tests import PsutilTestCase +from psutil.tests import bind_socket +from psutil.tests import bind_unix_socket +from psutil.tests import check_connection_ntuple +from psutil.tests import create_sockets +from psutil.tests import filter_proc_net_connections +from psutil.tests import pytest +from psutil.tests import reap_children +from psutil.tests import retry_on_failure +from psutil.tests import skip_on_access_denied +from psutil.tests import tcp_socketpair +from psutil.tests import unix_socketpair +from psutil.tests import wait_for_file + + +SOCK_SEQPACKET = getattr(socket, "SOCK_SEQPACKET", object()) + + +def this_proc_net_connections(kind): + cons = psutil.Process().net_connections(kind=kind) + if kind in {"all", "unix"}: + return filter_proc_net_connections(cons) + return cons + + +@pytest.mark.xdist_group(name="serial") +class ConnectionTestCase(PsutilTestCase): + def setUp(self): + assert this_proc_net_connections(kind='all') == [] + + def tearDown(self): + # Make sure we closed all resources. + assert this_proc_net_connections(kind='all') == [] + + def compare_procsys_connections(self, pid, proc_cons, kind='all'): + """Given a process PID and its list of connections compare + those against system-wide connections retrieved via + psutil.net_connections. + """ + try: + sys_cons = psutil.net_connections(kind=kind) + except psutil.AccessDenied: + # On MACOS, system-wide connections are retrieved by iterating + # over all processes + if MACOS: + return + else: + raise + # Filter for this proc PID and exlucde PIDs from the tuple. + sys_cons = [c[:-1] for c in sys_cons if c.pid == pid] + sys_cons.sort() + proc_cons.sort() + assert proc_cons == sys_cons + + +class TestBasicOperations(ConnectionTestCase): + @pytest.mark.skipif(SKIP_SYSCONS, reason="requires root") + def test_system(self): + with create_sockets(): + for conn in psutil.net_connections(kind='all'): + check_connection_ntuple(conn) + + def test_process(self): + with create_sockets(): + for conn in this_proc_net_connections(kind='all'): + check_connection_ntuple(conn) + + def test_invalid_kind(self): + with pytest.raises(ValueError): + this_proc_net_connections(kind='???') + with pytest.raises(ValueError): + psutil.net_connections(kind='???') + + +@pytest.mark.xdist_group(name="serial") +class TestUnconnectedSockets(ConnectionTestCase): + """Tests sockets which are open but not connected to anything.""" + + def get_conn_from_sock(self, sock): + cons = this_proc_net_connections(kind='all') + smap = {c.fd: c for c in cons} + if NETBSD or FREEBSD: + # NetBSD opens a UNIX socket to /var/log/run + # so there may be more connections. + return smap[sock.fileno()] + else: + assert len(cons) == 1 + if cons[0].fd != -1: + assert smap[sock.fileno()].fd == sock.fileno() + return cons[0] + + def check_socket(self, sock): + """Given a socket, makes sure it matches the one obtained + via psutil. It assumes this process created one connection + only (the one supposed to be checked). + """ + conn = self.get_conn_from_sock(sock) + check_connection_ntuple(conn) + + # fd, family, type + if conn.fd != -1: + assert conn.fd == sock.fileno() + assert conn.family == sock.family + # see: http://bugs.python.org/issue30204 + assert conn.type == sock.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE) + + # local address + laddr = sock.getsockname() + if not laddr and isinstance(laddr, bytes): + # See: http://bugs.python.org/issue30205 + laddr = laddr.decode() + if sock.family == AF_INET6: + laddr = laddr[:2] + assert conn.laddr == laddr + + # XXX Solaris can't retrieve system-wide UNIX sockets + if sock.family == AF_UNIX and HAS_NET_CONNECTIONS_UNIX: + cons = this_proc_net_connections(kind='all') + self.compare_procsys_connections(os.getpid(), cons, kind='all') + return conn + + def test_tcp_v4(self): + addr = ("127.0.0.1", 0) + with closing(bind_socket(AF_INET, SOCK_STREAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_LISTEN + + @pytest.mark.skipif(not supports_ipv6(), reason="IPv6 not supported") + def test_tcp_v6(self): + addr = ("::1", 0) + with closing(bind_socket(AF_INET6, SOCK_STREAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_LISTEN + + def test_udp_v4(self): + addr = ("127.0.0.1", 0) + with closing(bind_socket(AF_INET, SOCK_DGRAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_NONE + + @pytest.mark.skipif(not supports_ipv6(), reason="IPv6 not supported") + def test_udp_v6(self): + addr = ("::1", 0) + with closing(bind_socket(AF_INET6, SOCK_DGRAM, addr=addr)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == () + assert conn.status == psutil.CONN_NONE + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_unix_tcp(self): + testfn = self.get_testfn() + with closing(bind_unix_socket(testfn, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == "" + assert conn.status == psutil.CONN_NONE + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_unix_udp(self): + testfn = self.get_testfn() + with closing(bind_unix_socket(testfn, type=SOCK_STREAM)) as sock: + conn = self.check_socket(sock) + assert conn.raddr == "" + assert conn.status == psutil.CONN_NONE + + +@pytest.mark.xdist_group(name="serial") +class TestConnectedSocket(ConnectionTestCase): + """Test socket pairs which are actually connected to + each other. + """ + + # On SunOS, even after we close() it, the server socket stays around + # in TIME_WAIT state. + @pytest.mark.skipif(SUNOS, reason="unreliable on SUONS") + def test_tcp(self): + addr = ("127.0.0.1", 0) + assert this_proc_net_connections(kind='tcp4') == [] + server, client = tcp_socketpair(AF_INET, addr=addr) + try: + cons = this_proc_net_connections(kind='tcp4') + assert len(cons) == 2 + assert cons[0].status == psutil.CONN_ESTABLISHED + assert cons[1].status == psutil.CONN_ESTABLISHED + # May not be fast enough to change state so it stays + # commenteed. + # client.close() + # cons = this_proc_net_connections(kind='all') + # self.assertEqual(len(cons), 1) + # self.assertEqual(cons[0].status, psutil.CONN_CLOSE_WAIT) + finally: + server.close() + client.close() + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_unix(self): + testfn = self.get_testfn() + server, client = unix_socketpair(testfn) + try: + cons = this_proc_net_connections(kind='unix') + assert not (cons[0].laddr and cons[0].raddr), cons + assert not (cons[1].laddr and cons[1].raddr), cons + if NETBSD or FREEBSD: + # On NetBSD creating a UNIX socket will cause + # a UNIX connection to /var/run/log. + cons = [c for c in cons if c.raddr != '/var/run/log'] + assert len(cons) == 2 + if LINUX or FREEBSD or SUNOS or OPENBSD: + # remote path is never set + assert cons[0].raddr == "" + assert cons[1].raddr == "" + # one local address should though + assert testfn == (cons[0].laddr or cons[1].laddr) + else: + # On other systems either the laddr or raddr + # of both peers are set. + assert (cons[0].laddr or cons[1].laddr) == testfn + finally: + server.close() + client.close() + + +class TestFilters(ConnectionTestCase): + def test_filters(self): + def check(kind, families, types): + for conn in this_proc_net_connections(kind=kind): + assert conn.family in families + assert conn.type in types + if not SKIP_SYSCONS: + for conn in psutil.net_connections(kind=kind): + assert conn.family in families + assert conn.type in types + + with create_sockets(): + check( + 'all', + [AF_INET, AF_INET6, AF_UNIX], + [SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET], + ) + check('inet', [AF_INET, AF_INET6], [SOCK_STREAM, SOCK_DGRAM]) + check('inet4', [AF_INET], [SOCK_STREAM, SOCK_DGRAM]) + check('tcp', [AF_INET, AF_INET6], [SOCK_STREAM]) + check('tcp4', [AF_INET], [SOCK_STREAM]) + check('tcp6', [AF_INET6], [SOCK_STREAM]) + check('udp', [AF_INET, AF_INET6], [SOCK_DGRAM]) + check('udp4', [AF_INET], [SOCK_DGRAM]) + check('udp6', [AF_INET6], [SOCK_DGRAM]) + if HAS_NET_CONNECTIONS_UNIX: + check( + 'unix', + [AF_UNIX], + [SOCK_STREAM, SOCK_DGRAM, SOCK_SEQPACKET], + ) + + @skip_on_access_denied(only_if=MACOS) + def test_combos(self): + reap_children() + + def check_conn(proc, conn, family, type, laddr, raddr, status, kinds): + all_kinds = ( + "all", + "inet", + "inet4", + "inet6", + "tcp", + "tcp4", + "tcp6", + "udp", + "udp4", + "udp6", + ) + check_connection_ntuple(conn) + assert conn.family == family + assert conn.type == type + assert conn.laddr == laddr + assert conn.raddr == raddr + assert conn.status == status + for kind in all_kinds: + cons = proc.net_connections(kind=kind) + if kind in kinds: + assert cons != [] + else: + assert cons == [] + # compare against system-wide connections + # XXX Solaris can't retrieve system-wide UNIX + # sockets. + if HAS_NET_CONNECTIONS_UNIX: + self.compare_procsys_connections(proc.pid, [conn]) + + tcp_template = textwrap.dedent(""" + import socket, time + s = socket.socket({family}, socket.SOCK_STREAM) + s.bind(('{addr}', 0)) + s.listen(5) + with open('{testfn}', 'w') as f: + f.write(str(s.getsockname()[:2])) + [time.sleep(0.1) for x in range(100)] + """) + + udp_template = textwrap.dedent(""" + import socket, time + s = socket.socket({family}, socket.SOCK_DGRAM) + s.bind(('{addr}', 0)) + with open('{testfn}', 'w') as f: + f.write(str(s.getsockname()[:2])) + [time.sleep(0.1) for x in range(100)] + """) + + # must be relative on Windows + testfile = os.path.basename(self.get_testfn(dir=os.getcwd())) + tcp4_template = tcp_template.format( + family=int(AF_INET), addr="127.0.0.1", testfn=testfile + ) + udp4_template = udp_template.format( + family=int(AF_INET), addr="127.0.0.1", testfn=testfile + ) + tcp6_template = tcp_template.format( + family=int(AF_INET6), addr="::1", testfn=testfile + ) + udp6_template = udp_template.format( + family=int(AF_INET6), addr="::1", testfn=testfile + ) + + # launch various subprocess instantiating a socket of various + # families and types to enrich psutil results + tcp4_proc = self.pyrun(tcp4_template) + tcp4_addr = eval(wait_for_file(testfile, delete=True)) + udp4_proc = self.pyrun(udp4_template) + udp4_addr = eval(wait_for_file(testfile, delete=True)) + if supports_ipv6(): + tcp6_proc = self.pyrun(tcp6_template) + tcp6_addr = eval(wait_for_file(testfile, delete=True)) + udp6_proc = self.pyrun(udp6_template) + udp6_addr = eval(wait_for_file(testfile, delete=True)) + else: + tcp6_proc = None + udp6_proc = None + tcp6_addr = None + udp6_addr = None + + for p in psutil.Process().children(): + cons = p.net_connections() + assert len(cons) == 1 + for conn in cons: + # TCP v4 + if p.pid == tcp4_proc.pid: + check_conn( + p, + conn, + AF_INET, + SOCK_STREAM, + tcp4_addr, + (), + psutil.CONN_LISTEN, + ("all", "inet", "inet4", "tcp", "tcp4"), + ) + # UDP v4 + elif p.pid == udp4_proc.pid: + check_conn( + p, + conn, + AF_INET, + SOCK_DGRAM, + udp4_addr, + (), + psutil.CONN_NONE, + ("all", "inet", "inet4", "udp", "udp4"), + ) + # TCP v6 + elif p.pid == getattr(tcp6_proc, "pid", None): + check_conn( + p, + conn, + AF_INET6, + SOCK_STREAM, + tcp6_addr, + (), + psutil.CONN_LISTEN, + ("all", "inet", "inet6", "tcp", "tcp6"), + ) + # UDP v6 + elif p.pid == getattr(udp6_proc, "pid", None): + check_conn( + p, + conn, + AF_INET6, + SOCK_DGRAM, + udp6_addr, + (), + psutil.CONN_NONE, + ("all", "inet", "inet6", "udp", "udp6"), + ) + + def test_count(self): + with create_sockets(): + # tcp + cons = this_proc_net_connections(kind='tcp') + assert len(cons) == (2 if supports_ipv6() else 1) + for conn in cons: + assert conn.family in {AF_INET, AF_INET6} + assert conn.type == SOCK_STREAM + # tcp4 + cons = this_proc_net_connections(kind='tcp4') + assert len(cons) == 1 + assert cons[0].family == AF_INET + assert cons[0].type == SOCK_STREAM + # tcp6 + if supports_ipv6(): + cons = this_proc_net_connections(kind='tcp6') + assert len(cons) == 1 + assert cons[0].family == AF_INET6 + assert cons[0].type == SOCK_STREAM + # udp + cons = this_proc_net_connections(kind='udp') + assert len(cons) == (2 if supports_ipv6() else 1) + for conn in cons: + assert conn.family in {AF_INET, AF_INET6} + assert conn.type == SOCK_DGRAM + # udp4 + cons = this_proc_net_connections(kind='udp4') + assert len(cons) == 1 + assert cons[0].family == AF_INET + assert cons[0].type == SOCK_DGRAM + # udp6 + if supports_ipv6(): + cons = this_proc_net_connections(kind='udp6') + assert len(cons) == 1 + assert cons[0].family == AF_INET6 + assert cons[0].type == SOCK_DGRAM + # inet + cons = this_proc_net_connections(kind='inet') + assert len(cons) == (4 if supports_ipv6() else 2) + for conn in cons: + assert conn.family in {AF_INET, AF_INET6} + assert conn.type in {SOCK_STREAM, SOCK_DGRAM} + # inet6 + if supports_ipv6(): + cons = this_proc_net_connections(kind='inet6') + assert len(cons) == 2 + for conn in cons: + assert conn.family == AF_INET6 + assert conn.type in {SOCK_STREAM, SOCK_DGRAM} + # Skipped on BSD becayse by default the Python process + # creates a UNIX socket to '/var/run/log'. + if HAS_NET_CONNECTIONS_UNIX and not (FREEBSD or NETBSD): + cons = this_proc_net_connections(kind='unix') + assert len(cons) == 3 + for conn in cons: + assert conn.family == AF_UNIX + assert conn.type in {SOCK_STREAM, SOCK_DGRAM} + + +@pytest.mark.skipif(SKIP_SYSCONS, reason="requires root") +class TestSystemWideConnections(ConnectionTestCase): + """Tests for net_connections().""" + + def test_it(self): + def check(cons, families, types_): + for conn in cons: + assert conn.family in families + if conn.family != AF_UNIX: + assert conn.type in types_ + check_connection_ntuple(conn) + + with create_sockets(): + from psutil._common import conn_tmap + + for kind, groups in conn_tmap.items(): + # XXX: SunOS does not retrieve UNIX sockets. + if kind == 'unix' and not HAS_NET_CONNECTIONS_UNIX: + continue + families, types_ = groups + cons = psutil.net_connections(kind) + assert len(cons) == len(set(cons)) + check(cons, families, types_) + + @retry_on_failure() + def test_multi_sockets_procs(self): + # Creates multiple sub processes, each creating different + # sockets. For each process check that proc.net_connections() + # and psutil.net_connections() return the same results. + # This is done mainly to check whether net_connections()'s + # pid is properly set, see: + # https://github.com/giampaolo/psutil/issues/1013 + with create_sockets() as socks: + expected = len(socks) + pids = [] + times = 10 + fnames = [] + for _ in range(times): + fname = self.get_testfn() + fnames.append(fname) + src = textwrap.dedent(f"""\ + import time, os + from psutil.tests import create_sockets + with create_sockets(): + with open(r'{fname}', 'w') as f: + f.write("hello") + [time.sleep(0.1) for x in range(100)] + """) + sproc = self.pyrun(src) + pids.append(sproc.pid) + + # sync + for fname in fnames: + wait_for_file(fname) + + syscons = [ + x for x in psutil.net_connections(kind='all') if x.pid in pids + ] + for pid in pids: + assert len([x for x in syscons if x.pid == pid]) == expected + p = psutil.Process(pid) + assert len(p.net_connections('all')) == expected + + +class TestMisc(PsutilTestCase): + def test_net_connection_constants(self): + ints = [] + strs = [] + for name in dir(psutil): + if name.startswith('CONN_'): + num = getattr(psutil, name) + str_ = str(num) + assert str_.isupper(), str_ + assert str not in strs + assert num not in ints + ints.append(num) + strs.append(str_) + if SUNOS: + psutil.CONN_IDLE # noqa: B018 + psutil.CONN_BOUND # noqa: B018 + if WINDOWS: + psutil.CONN_DELETE_TCB # noqa: B018 diff --git a/lib/python3.12/site-packages/psutil/tests/test_contracts.py b/lib/python3.12/site-packages/psutil/tests/test_contracts.py new file mode 100644 index 0000000000000000000000000000000000000000..55f3a5ddb82beb9363e92eb4a2d61f3fac9ea329 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_contracts.py @@ -0,0 +1,325 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Contracts tests. These tests mainly check API sanity in terms of +returned types and APIs availability. +Some of these are duplicates of tests test_system.py and test_process.py. +""" + +import platform +import signal + +import psutil +from psutil import AIX +from psutil import FREEBSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_NET_IO_COUNTERS +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import SKIP_SYSCONS +from psutil.tests import PsutilTestCase +from psutil.tests import create_sockets +from psutil.tests import enum +from psutil.tests import is_namedtuple +from psutil.tests import kernel_version +from psutil.tests import pytest + + +# =================================================================== +# --- APIs availability +# =================================================================== + +# Make sure code reflects what doc promises in terms of APIs +# availability. + + +class TestAvailConstantsAPIs(PsutilTestCase): + def test_PROCFS_PATH(self): + assert hasattr(psutil, "PROCFS_PATH") == (LINUX or SUNOS or AIX) + + def test_win_priority(self): + ae = self.assertEqual + ae(hasattr(psutil, "ABOVE_NORMAL_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "BELOW_NORMAL_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "HIGH_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "IDLE_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "NORMAL_PRIORITY_CLASS"), WINDOWS) + ae(hasattr(psutil, "REALTIME_PRIORITY_CLASS"), WINDOWS) + + def test_linux_ioprio_linux(self): + ae = self.assertEqual + ae(hasattr(psutil, "IOPRIO_CLASS_NONE"), LINUX) + ae(hasattr(psutil, "IOPRIO_CLASS_RT"), LINUX) + ae(hasattr(psutil, "IOPRIO_CLASS_BE"), LINUX) + ae(hasattr(psutil, "IOPRIO_CLASS_IDLE"), LINUX) + + def test_linux_ioprio_windows(self): + ae = self.assertEqual + ae(hasattr(psutil, "IOPRIO_HIGH"), WINDOWS) + ae(hasattr(psutil, "IOPRIO_NORMAL"), WINDOWS) + ae(hasattr(psutil, "IOPRIO_LOW"), WINDOWS) + ae(hasattr(psutil, "IOPRIO_VERYLOW"), WINDOWS) + + @pytest.mark.skipif( + GITHUB_ACTIONS and LINUX, + reason="unsupported on GITHUB_ACTIONS + LINUX", + ) + def test_rlimit(self): + ae = self.assertEqual + ae(hasattr(psutil, "RLIM_INFINITY"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_AS"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_CORE"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_CPU"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_DATA"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_FSIZE"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_MEMLOCK"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_NOFILE"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_NPROC"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_RSS"), LINUX or FREEBSD) + ae(hasattr(psutil, "RLIMIT_STACK"), LINUX or FREEBSD) + + ae(hasattr(psutil, "RLIMIT_LOCKS"), LINUX) + if POSIX: + if kernel_version() >= (2, 6, 8): + ae(hasattr(psutil, "RLIMIT_MSGQUEUE"), LINUX) + if kernel_version() >= (2, 6, 12): + ae(hasattr(psutil, "RLIMIT_NICE"), LINUX) + if kernel_version() >= (2, 6, 12): + ae(hasattr(psutil, "RLIMIT_RTPRIO"), LINUX) + if kernel_version() >= (2, 6, 25): + ae(hasattr(psutil, "RLIMIT_RTTIME"), LINUX) + if kernel_version() >= (2, 6, 8): + ae(hasattr(psutil, "RLIMIT_SIGPENDING"), LINUX) + + ae(hasattr(psutil, "RLIMIT_SWAP"), FREEBSD) + ae(hasattr(psutil, "RLIMIT_SBSIZE"), FREEBSD) + ae(hasattr(psutil, "RLIMIT_NPTS"), FREEBSD) + + +class TestAvailSystemAPIs(PsutilTestCase): + def test_win_service_iter(self): + assert hasattr(psutil, "win_service_iter") == WINDOWS + + def test_win_service_get(self): + assert hasattr(psutil, "win_service_get") == WINDOWS + + def test_cpu_freq(self): + assert hasattr(psutil, "cpu_freq") == ( + LINUX or MACOS or WINDOWS or FREEBSD or OPENBSD + ) + + def test_sensors_temperatures(self): + assert hasattr(psutil, "sensors_temperatures") == (LINUX or FREEBSD) + + def test_sensors_fans(self): + assert hasattr(psutil, "sensors_fans") == LINUX + + def test_battery(self): + assert hasattr(psutil, "sensors_battery") == ( + LINUX or WINDOWS or FREEBSD or MACOS + ) + + +class TestAvailProcessAPIs(PsutilTestCase): + def test_environ(self): + assert hasattr(psutil.Process, "environ") == ( + LINUX + or MACOS + or WINDOWS + or AIX + or SUNOS + or FREEBSD + or OPENBSD + or NETBSD + ) + + def test_uids(self): + assert hasattr(psutil.Process, "uids") == POSIX + + def test_gids(self): + assert hasattr(psutil.Process, "uids") == POSIX + + def test_terminal(self): + assert hasattr(psutil.Process, "terminal") == POSIX + + def test_ionice(self): + assert hasattr(psutil.Process, "ionice") == (LINUX or WINDOWS) + + @pytest.mark.skipif( + GITHUB_ACTIONS and LINUX, + reason="unsupported on GITHUB_ACTIONS + LINUX", + ) + def test_rlimit(self): + assert hasattr(psutil.Process, "rlimit") == (LINUX or FREEBSD) + + def test_io_counters(self): + hasit = hasattr(psutil.Process, "io_counters") + assert hasit == (not (MACOS or SUNOS)) + + def test_num_fds(self): + assert hasattr(psutil.Process, "num_fds") == POSIX + + def test_num_handles(self): + assert hasattr(psutil.Process, "num_handles") == WINDOWS + + def test_cpu_affinity(self): + assert hasattr(psutil.Process, "cpu_affinity") == ( + LINUX or WINDOWS or FREEBSD + ) + + def test_cpu_num(self): + assert hasattr(psutil.Process, "cpu_num") == ( + LINUX or FREEBSD or SUNOS + ) + + def test_memory_maps(self): + hasit = hasattr(psutil.Process, "memory_maps") + assert hasit == (not (OPENBSD or NETBSD or AIX or MACOS)) + + +# =================================================================== +# --- API types +# =================================================================== + + +class TestSystemAPITypes(PsutilTestCase): + """Check the return types of system related APIs. + https://github.com/giampaolo/psutil/issues/1039. + """ + + @classmethod + def setUpClass(cls): + cls.proc = psutil.Process() + + def assert_ntuple_of_nums(self, nt, type_=float, gezero=True): + assert is_namedtuple(nt) + for n in nt: + assert isinstance(n, type_) + if gezero: + assert n >= 0 + + def test_cpu_times(self): + self.assert_ntuple_of_nums(psutil.cpu_times()) + for nt in psutil.cpu_times(percpu=True): + self.assert_ntuple_of_nums(nt) + + def test_cpu_percent(self): + assert isinstance(psutil.cpu_percent(interval=None), float) + assert isinstance(psutil.cpu_percent(interval=0.00001), float) + + def test_cpu_times_percent(self): + self.assert_ntuple_of_nums(psutil.cpu_times_percent(interval=None)) + self.assert_ntuple_of_nums(psutil.cpu_times_percent(interval=0.0001)) + + def test_cpu_count(self): + assert isinstance(psutil.cpu_count(), int) + + # TODO: remove this once 1892 is fixed + @pytest.mark.skipif( + MACOS and platform.machine() == 'arm64', reason="skipped due to #1892" + ) + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + def test_cpu_freq(self): + if psutil.cpu_freq() is None: + raise pytest.skip("cpu_freq() returns None") + self.assert_ntuple_of_nums(psutil.cpu_freq(), type_=(float, int)) + + def test_disk_io_counters(self): + # Duplicate of test_system.py. Keep it anyway. + for k, v in psutil.disk_io_counters(perdisk=True).items(): + assert isinstance(k, str) + self.assert_ntuple_of_nums(v, type_=int) + + def test_disk_partitions(self): + # Duplicate of test_system.py. Keep it anyway. + for disk in psutil.disk_partitions(): + assert isinstance(disk.device, str) + assert isinstance(disk.mountpoint, str) + assert isinstance(disk.fstype, str) + assert isinstance(disk.opts, str) + + @pytest.mark.skipif(SKIP_SYSCONS, reason="requires root") + def test_net_connections(self): + with create_sockets(): + ret = psutil.net_connections('all') + assert len(ret) == len(set(ret)) + for conn in ret: + assert is_namedtuple(conn) + + def test_net_if_addrs(self): + # Duplicate of test_system.py. Keep it anyway. + for ifname, addrs in psutil.net_if_addrs().items(): + assert isinstance(ifname, str) + for addr in addrs: + assert isinstance(addr.family, enum.IntEnum) + assert isinstance(addr.address, str) + assert isinstance(addr.netmask, (str, type(None))) + assert isinstance(addr.broadcast, (str, type(None))) + + def test_net_if_stats(self): + # Duplicate of test_system.py. Keep it anyway. + for ifname, info in psutil.net_if_stats().items(): + assert isinstance(ifname, str) + assert isinstance(info.isup, bool) + assert isinstance(info.duplex, enum.IntEnum) + assert isinstance(info.speed, int) + assert isinstance(info.mtu, int) + + @pytest.mark.skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_net_io_counters(self): + # Duplicate of test_system.py. Keep it anyway. + for ifname in psutil.net_io_counters(pernic=True): + assert isinstance(ifname, str) + + @pytest.mark.skipif(not HAS_SENSORS_FANS, reason="not supported") + def test_sensors_fans(self): + # Duplicate of test_system.py. Keep it anyway. + for name, units in psutil.sensors_fans().items(): + assert isinstance(name, str) + for unit in units: + assert isinstance(unit.label, str) + assert isinstance(unit.current, (float, int, type(None))) + + @pytest.mark.skipif(not HAS_SENSORS_TEMPERATURES, reason="not supported") + def test_sensors_temperatures(self): + # Duplicate of test_system.py. Keep it anyway. + for name, units in psutil.sensors_temperatures().items(): + assert isinstance(name, str) + for unit in units: + assert isinstance(unit.label, str) + assert isinstance(unit.current, (float, int, type(None))) + assert isinstance(unit.high, (float, int, type(None))) + assert isinstance(unit.critical, (float, int, type(None))) + + def test_boot_time(self): + # Duplicate of test_system.py. Keep it anyway. + assert isinstance(psutil.boot_time(), float) + + def test_users(self): + # Duplicate of test_system.py. Keep it anyway. + for user in psutil.users(): + assert isinstance(user.name, str) + assert isinstance(user.terminal, (str, type(None))) + assert isinstance(user.host, (str, type(None))) + assert isinstance(user.pid, (int, type(None))) + + +class TestProcessWaitType(PsutilTestCase): + @pytest.mark.skipif(not POSIX, reason="not POSIX") + def test_negative_signal(self): + p = psutil.Process(self.spawn_testproc().pid) + p.terminate() + code = p.wait() + assert code == -signal.SIGTERM + assert isinstance(code, enum.IntEnum) diff --git a/lib/python3.12/site-packages/psutil/tests/test_linux.py b/lib/python3.12/site-packages/psutil/tests/test_linux.py new file mode 100644 index 0000000000000000000000000000000000000000..f4342d7aa34a74168e74f6fcb306d5b59ea638b6 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_linux.py @@ -0,0 +1,2292 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Linux specific tests.""" + + +import collections +import contextlib +import errno +import io +import os +import platform +import re +import shutil +import socket +import struct +import textwrap +import time +import warnings +from unittest import mock + +import psutil +from psutil import LINUX +from psutil.tests import AARCH64 +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import GLOBAL_TIMEOUT +from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_GETLOADAVG +from psutil.tests import HAS_RLIMIT +from psutil.tests import PYPY +from psutil.tests import PYTEST_PARALLEL +from psutil.tests import TOLERANCE_DISK_USAGE +from psutil.tests import TOLERANCE_SYS_MEM +from psutil.tests import PsutilTestCase +from psutil.tests import ThreadTask +from psutil.tests import call_until +from psutil.tests import pytest +from psutil.tests import reload_module +from psutil.tests import retry_on_failure +from psutil.tests import safe_rmpath +from psutil.tests import sh +from psutil.tests import skip_on_not_implemented + + +if LINUX: + from psutil._pslinux import CLOCK_TICKS + from psutil._pslinux import RootFsDeviceFinder + from psutil._pslinux import calculate_avail_vmem + from psutil._pslinux import open_binary + + +HERE = os.path.abspath(os.path.dirname(__file__)) +SIOCGIFADDR = 0x8915 +SIOCGIFHWADDR = 0x8927 +SIOCGIFNETMASK = 0x891B +SIOCGIFBRDADDR = 0x8919 +if LINUX: + SECTOR_SIZE = 512 +# ===================================================================== +# --- utils +# ===================================================================== + + +def get_ipv4_address(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + return socket.inet_ntoa( + fcntl.ioctl(s.fileno(), SIOCGIFADDR, struct.pack('256s', ifname))[ + 20:24 + ] + ) + + +def get_ipv4_netmask(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + return socket.inet_ntoa( + fcntl.ioctl( + s.fileno(), SIOCGIFNETMASK, struct.pack('256s', ifname) + )[20:24] + ) + + +def get_ipv4_broadcast(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + return socket.inet_ntoa( + fcntl.ioctl( + s.fileno(), SIOCGIFBRDADDR, struct.pack('256s', ifname) + )[20:24] + ) + + +def get_ipv6_addresses(ifname): + with open("/proc/net/if_inet6") as f: + all_fields = [] + for line in f: + fields = line.split() + if fields[-1] == ifname: + all_fields.append(fields) + + if len(all_fields) == 0: + raise ValueError(f"could not find interface {ifname!r}") + + for i in range(len(all_fields)): + unformatted = all_fields[i][0] + groups = [ + unformatted[j : j + 4] for j in range(0, len(unformatted), 4) + ] + formatted = ":".join(groups) + packed = socket.inet_pton(socket.AF_INET6, formatted) + all_fields[i] = socket.inet_ntop(socket.AF_INET6, packed) + return all_fields + + +def get_mac_address(ifname): + import fcntl + + ifname = bytes(ifname[:15], "ascii") + with socket.socket(socket.AF_INET, socket.SOCK_DGRAM) as s: + info = fcntl.ioctl( + s.fileno(), SIOCGIFHWADDR, struct.pack('256s', ifname) + ) + return "".join([f"{char:02x}:" for char in info[18:24]])[:-1] + + +def free_swap(): + """Parse 'free' cmd and return swap memory's s total, used and free + values. + """ + out = sh(["free", "-b"], env={"LANG": "C.UTF-8"}) + lines = out.split('\n') + for line in lines: + if line.startswith('Swap'): + _, total, used, free = line.split() + nt = collections.namedtuple('free', 'total used free') + return nt(int(total), int(used), int(free)) + raise ValueError(f"can't find 'Swap' in 'free' output:\n{out}") + + +def free_physmem(): + """Parse 'free' cmd and return physical memory's total, used + and free values. + """ + # Note: free can have 2 different formats, invalidating 'shared' + # and 'cached' memory which may have different positions so we + # do not return them. + # https://github.com/giampaolo/psutil/issues/538#issuecomment-57059946 + out = sh(["free", "-b"], env={"LANG": "C.UTF-8"}) + lines = out.split('\n') + for line in lines: + if line.startswith('Mem'): + total, used, free, shared = (int(x) for x in line.split()[1:5]) + nt = collections.namedtuple( + 'free', 'total used free shared output' + ) + return nt(total, used, free, shared, out) + raise ValueError(f"can't find 'Mem' in 'free' output:\n{out}") + + +def vmstat(stat): + out = sh(["vmstat", "-s"], env={"LANG": "C.UTF-8"}) + for line in out.split("\n"): + line = line.strip() + if stat in line: + return int(line.split(' ')[0]) + raise ValueError(f"can't find {stat!r} in 'vmstat' output") + + +def get_free_version_info(): + out = sh(["free", "-V"]).strip() + if 'UNKNOWN' in out: + raise pytest.skip("can't determine free version") + return tuple(map(int, re.findall(r'\d+', out.split()[-1]))) + + +@contextlib.contextmanager +def mock_open_content(pairs): + """Mock open() builtin and forces it to return a certain content + for a given path. `pairs` is a {"path": "content", ...} dict. + """ + + def open_mock(name, *args, **kwargs): + if name in pairs: + content = pairs[name] + if isinstance(content, str): + return io.StringIO(content) + else: + return io.BytesIO(content) + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", create=True, side_effect=open_mock) as m: + yield m + + +@contextlib.contextmanager +def mock_open_exception(for_path, exc): + """Mock open() builtin and raises `exc` if the path being opened + matches `for_path`. + """ + + def open_mock(name, *args, **kwargs): + if name == for_path: + raise exc + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", create=True, side_effect=open_mock) as m: + yield m + + +# ===================================================================== +# --- system virtual memory +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemVirtualMemoryAgainstFree(PsutilTestCase): + def test_total(self): + cli_value = free_physmem().total + psutil_value = psutil.virtual_memory().total + assert cli_value == psutil_value + + @retry_on_failure() + def test_used(self): + # Older versions of procps used slab memory to calculate used memory. + # This got changed in: + # https://gitlab.com/procps-ng/procps/commit/ + # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + # Newer versions of procps are using yet another way to compute used + # memory. + # https://gitlab.com/procps-ng/procps/commit/ + # 2184e90d2e7cdb582f9a5b706b47015e56707e4d + if get_free_version_info() < (3, 3, 12): + raise pytest.skip("free version too old") + if get_free_version_info() >= (4, 0, 0): + raise pytest.skip("free version too recent") + cli_value = free_physmem().used + psutil_value = psutil.virtual_memory().used + assert abs(cli_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_free(self): + cli_value = free_physmem().free + psutil_value = psutil.virtual_memory().free + assert abs(cli_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_shared(self): + free = free_physmem() + free_value = free.shared + if free_value == 0: + raise pytest.skip("free does not support 'shared' column") + psutil_value = psutil.virtual_memory().shared + assert ( + abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + ), f"{free_value} {psutil_value} \n{free.output}" + + @retry_on_failure() + def test_available(self): + # "free" output format has changed at some point: + # https://github.com/giampaolo/psutil/issues/538#issuecomment-147192098 + out = sh(["free", "-b"]) + lines = out.split('\n') + if 'available' not in lines[0]: + raise pytest.skip("free does not support 'available' column") + free_value = int(lines[1].split()[-1]) + psutil_value = psutil.virtual_memory().available + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemVirtualMemoryAgainstVmstat(PsutilTestCase): + def test_total(self): + vmstat_value = vmstat('total memory') * 1024 + psutil_value = psutil.virtual_memory().total + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_used(self): + # Older versions of procps used slab memory to calculate used memory. + # This got changed in: + # https://gitlab.com/procps-ng/procps/commit/ + # 05d751c4f076a2f0118b914c5e51cfbb4762ad8e + # Newer versions of procps are using yet another way to compute used + # memory. + # https://gitlab.com/procps-ng/procps/commit/ + # 2184e90d2e7cdb582f9a5b706b47015e56707e4d + if get_free_version_info() < (3, 3, 12): + raise pytest.skip("free version too old") + if get_free_version_info() >= (4, 0, 0): + raise pytest.skip("free version too recent") + vmstat_value = vmstat('used memory') * 1024 + psutil_value = psutil.virtual_memory().used + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_free(self): + vmstat_value = vmstat('free memory') * 1024 + psutil_value = psutil.virtual_memory().free + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_buffers(self): + vmstat_value = vmstat('buffer memory') * 1024 + psutil_value = psutil.virtual_memory().buffers + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_active(self): + vmstat_value = vmstat('active memory') * 1024 + psutil_value = psutil.virtual_memory().active + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_inactive(self): + vmstat_value = vmstat('inactive memory') * 1024 + psutil_value = psutil.virtual_memory().inactive + assert abs(vmstat_value - psutil_value) < TOLERANCE_SYS_MEM + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemVirtualMemoryMocks(PsutilTestCase): + def test_warnings_on_misses(self): + # Emulate a case where /proc/meminfo provides few info. + # psutil is supposed to set the missing fields to 0 and + # raise a warning. + content = textwrap.dedent("""\ + Active(anon): 6145416 kB + Active(file): 2950064 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: -1 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + SReclaimable: 346648 kB + """).encode() + with mock_open_content({'/proc/meminfo': content}) as m: + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + ret = psutil.virtual_memory() + assert m.called + assert len(ws) == 1 + w = ws[0] + assert "memory stats couldn't be determined" in str(w.message) + assert "cached" in str(w.message) + assert "shared" in str(w.message) + assert "active" in str(w.message) + assert "inactive" in str(w.message) + assert "buffers" in str(w.message) + assert "available" in str(w.message) + assert ret.cached == 0 + assert ret.active == 0 + assert ret.inactive == 0 + assert ret.shared == 0 + assert ret.buffers == 0 + assert ret.available == 0 + assert ret.slab == 0 + + @retry_on_failure() + def test_avail_old_percent(self): + # Make sure that our calculation of avail mem for old kernels + # is off by max 15%. + mems = {} + with open_binary('/proc/meminfo') as f: + for line in f: + fields = line.split() + mems[fields[0]] = int(fields[1]) * 1024 + + a = calculate_avail_vmem(mems) + if b'MemAvailable:' in mems: + b = mems[b'MemAvailable:'] + diff_percent = abs(a - b) / a * 100 + assert diff_percent < 15 + + def test_avail_old_comes_from_kernel(self): + # Make sure "MemAvailable:" coluimn is used instead of relying + # on our internal algorithm to calculate avail mem. + content = textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemAvailable: 6574984 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + SReclaimable: 346648 kB + """).encode() + with mock_open_content({'/proc/meminfo': content}) as m: + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() + assert m.called + assert ret.available == 6574984 * 1024 + w = ws[0] + assert "inactive memory stats couldn't be determined" in str( + w.message + ) + + def test_avail_old_missing_fields(self): + # Remove Active(file), Inactive(file) and SReclaimable + # from /proc/meminfo and make sure the fallback is used + # (free + cached), + content = textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}) as m: + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() + assert m.called + assert ret.available == 2057400 * 1024 + 4818144 * 1024 + w = ws[0] + assert "inactive memory stats couldn't be determined" in str( + w.message + ) + + def test_avail_old_missing_zoneinfo(self): + # Remove /proc/zoneinfo file. Make sure fallback is used + # (free + cached). + content = textwrap.dedent("""\ + Active: 9444728 kB + Active(anon): 6145416 kB + Active(file): 2950064 kB + Buffers: 287952 kB + Cached: 4818144 kB + Inactive(file): 1578132 kB + Inactive(anon): 574764 kB + Inactive(file): 1567648 kB + MemFree: 2057400 kB + MemTotal: 16325648 kB + Shmem: 577588 kB + SReclaimable: 346648 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}): + with mock_open_exception("/proc/zoneinfo", FileNotFoundError): + with warnings.catch_warnings(record=True) as ws: + ret = psutil.virtual_memory() + assert ret.available == 2057400 * 1024 + 4818144 * 1024 + w = ws[0] + assert ( + "inactive memory stats couldn't be determined" + in str(w.message) + ) + + def test_virtual_memory_mocked(self): + # Emulate /proc/meminfo because neither vmstat nor free return slab. + content = textwrap.dedent("""\ + MemTotal: 100 kB + MemFree: 2 kB + MemAvailable: 3 kB + Buffers: 4 kB + Cached: 5 kB + SwapCached: 6 kB + Active: 7 kB + Inactive: 8 kB + Active(anon): 9 kB + Inactive(anon): 10 kB + Active(file): 11 kB + Inactive(file): 12 kB + Unevictable: 13 kB + Mlocked: 14 kB + SwapTotal: 15 kB + SwapFree: 16 kB + Dirty: 17 kB + Writeback: 18 kB + AnonPages: 19 kB + Mapped: 20 kB + Shmem: 21 kB + Slab: 22 kB + SReclaimable: 23 kB + SUnreclaim: 24 kB + KernelStack: 25 kB + PageTables: 26 kB + NFS_Unstable: 27 kB + Bounce: 28 kB + WritebackTmp: 29 kB + CommitLimit: 30 kB + Committed_AS: 31 kB + VmallocTotal: 32 kB + VmallocUsed: 33 kB + VmallocChunk: 34 kB + HardwareCorrupted: 35 kB + AnonHugePages: 36 kB + ShmemHugePages: 37 kB + ShmemPmdMapped: 38 kB + CmaTotal: 39 kB + CmaFree: 40 kB + HugePages_Total: 41 kB + HugePages_Free: 42 kB + HugePages_Rsvd: 43 kB + HugePages_Surp: 44 kB + Hugepagesize: 45 kB + DirectMap46k: 46 kB + DirectMap47M: 47 kB + DirectMap48G: 48 kB + """).encode() + with mock_open_content({"/proc/meminfo": content}) as m: + mem = psutil.virtual_memory() + assert m.called + assert mem.total == 100 * 1024 + assert mem.free == 2 * 1024 + assert mem.buffers == 4 * 1024 + # cached mem also includes reclaimable memory + assert mem.cached == (5 + 23) * 1024 + assert mem.shared == 21 * 1024 + assert mem.active == 7 * 1024 + assert mem.inactive == 8 * 1024 + assert mem.slab == 22 * 1024 + assert mem.available == 3 * 1024 + + +# ===================================================================== +# --- system swap memory +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemSwapMemory(PsutilTestCase): + @staticmethod + def meminfo_has_swap_info(): + """Return True if /proc/meminfo provides swap metrics.""" + with open("/proc/meminfo") as f: + data = f.read() + return 'SwapTotal:' in data and 'SwapFree:' in data + + def test_total(self): + free_value = free_swap().total + psutil_value = psutil.swap_memory().total + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_used(self): + free_value = free_swap().used + psutil_value = psutil.swap_memory().used + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_free(self): + free_value = free_swap().free + psutil_value = psutil.swap_memory().free + assert abs(free_value - psutil_value) < TOLERANCE_SYS_MEM + + def test_missing_sin_sout(self): + with mock.patch('psutil._common.open', create=True) as m: + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + ret = psutil.swap_memory() + assert m.called + assert len(ws) == 1 + w = ws[0] + assert ( + "'sin' and 'sout' swap memory stats couldn't be determined" + in str(w.message) + ) + assert ret.sin == 0 + assert ret.sout == 0 + + def test_no_vmstat_mocked(self): + # see https://github.com/giampaolo/psutil/issues/722 + with mock_open_exception("/proc/vmstat", FileNotFoundError) as m: + with warnings.catch_warnings(record=True) as ws: + warnings.simplefilter("always") + ret = psutil.swap_memory() + assert m.called + assert len(ws) == 1 + w = ws[0] + assert ( + "'sin' and 'sout' swap memory stats couldn't " + "be determined and were set to 0" + in str(w.message) + ) + assert ret.sin == 0 + assert ret.sout == 0 + + def test_meminfo_against_sysinfo(self): + # Make sure the content of /proc/meminfo about swap memory + # matches sysinfo() syscall, see: + # https://github.com/giampaolo/psutil/issues/1015 + if not self.meminfo_has_swap_info(): + raise pytest.skip("/proc/meminfo has no swap metrics") + with mock.patch('psutil._pslinux.cext.linux_sysinfo') as m: + swap = psutil.swap_memory() + assert not m.called + import psutil._psutil_linux as cext + + _, _, _, _, total, free, unit_multiplier = cext.linux_sysinfo() + total *= unit_multiplier + free *= unit_multiplier + assert swap.total == total + assert abs(swap.free - free) < TOLERANCE_SYS_MEM + + def test_emulate_meminfo_has_no_metrics(self): + # Emulate a case where /proc/meminfo provides no swap metrics + # in which case sysinfo() syscall is supposed to be used + # as a fallback. + with mock_open_content({"/proc/meminfo": b""}) as m: + psutil.swap_memory() + assert m.called + + +# ===================================================================== +# --- system CPU +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemCPUTimes(PsutilTestCase): + def test_fields(self): + fields = psutil.cpu_times()._fields + kernel_ver = re.findall(r'\d+\.\d+\.\d+', os.uname()[2])[0] + kernel_ver_info = tuple(map(int, kernel_ver.split('.'))) + if kernel_ver_info >= (2, 6, 11): + assert 'steal' in fields + else: + assert 'steal' not in fields + if kernel_ver_info >= (2, 6, 24): + assert 'guest' in fields + else: + assert 'guest' not in fields + if kernel_ver_info >= (3, 2, 0): + assert 'guest_nice' in fields + else: + assert 'guest_nice' not in fields + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemCPUCountLogical(PsutilTestCase): + @pytest.mark.skipif( + not os.path.exists("/sys/devices/system/cpu/online"), + reason="/sys/devices/system/cpu/online does not exist", + ) + def test_against_sysdev_cpu_online(self): + with open("/sys/devices/system/cpu/online") as f: + value = f.read().strip() + if "-" in str(value): + value = int(value.split('-')[1]) + 1 + assert psutil.cpu_count() == value + + @pytest.mark.skipif( + not os.path.exists("/sys/devices/system/cpu"), + reason="/sys/devices/system/cpu does not exist", + ) + def test_against_sysdev_cpu_num(self): + ls = os.listdir("/sys/devices/system/cpu") + count = len([x for x in ls if re.search(r"cpu\d+$", x) is not None]) + assert psutil.cpu_count() == count + + @pytest.mark.skipif( + not shutil.which("nproc"), reason="nproc utility not available" + ) + def test_against_nproc(self): + num = int(sh("nproc --all")) + assert psutil.cpu_count(logical=True) == num + + @pytest.mark.skipif( + not shutil.which("lscpu"), reason="lscpu utility not available" + ) + def test_against_lscpu(self): + out = sh("lscpu -p") + num = len([x for x in out.split('\n') if not x.startswith('#')]) + assert psutil.cpu_count(logical=True) == num + + def test_emulate_fallbacks(self): + import psutil._pslinux + + original = psutil._pslinux.cpu_count_logical() + # Here we want to mock os.sysconf("SC_NPROCESSORS_ONLN") in + # order to cause the parsing of /proc/cpuinfo and /proc/stat. + with mock.patch( + 'psutil._pslinux.os.sysconf', side_effect=ValueError + ) as m: + assert psutil._pslinux.cpu_count_logical() == original + assert m.called + + # Let's have open() return empty data and make sure None is + # returned ('cause we mimic os.cpu_count()). + with mock.patch('psutil._common.open', create=True) as m: + assert psutil._pslinux.cpu_count_logical() is None + assert m.call_count == 2 + # /proc/stat should be the last one + assert m.call_args[0][0] == '/proc/stat' + + # Let's push this a bit further and make sure /proc/cpuinfo + # parsing works as expected. + with open('/proc/cpuinfo', 'rb') as f: + cpuinfo_data = f.read() + fake_file = io.BytesIO(cpuinfo_data) + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert psutil._pslinux.cpu_count_logical() == original + + # Finally, let's make /proc/cpuinfo return meaningless data; + # this way we'll fall back on relying on /proc/stat + with mock_open_content({"/proc/cpuinfo": b""}) as m: + assert psutil._pslinux.cpu_count_logical() == original + assert m.called + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemCPUCountCores(PsutilTestCase): + @pytest.mark.skipif( + not shutil.which("lscpu"), reason="lscpu utility not available" + ) + def test_against_lscpu(self): + out = sh("lscpu -p") + core_ids = set() + for line in out.split('\n'): + if not line.startswith('#'): + fields = line.split(',') + core_ids.add(fields[1]) + assert psutil.cpu_count(logical=False) == len(core_ids) + + @pytest.mark.skipif( + platform.machine() not in {"x86_64", "i686"}, reason="x86_64/i686 only" + ) + def test_method_2(self): + meth_1 = psutil._pslinux.cpu_count_cores() + with mock.patch('glob.glob', return_value=[]) as m: + meth_2 = psutil._pslinux.cpu_count_cores() + assert m.called + if meth_1 is not None: + assert meth_1 == meth_2 + + def test_emulate_none(self): + with mock.patch('glob.glob', return_value=[]) as m1: + with mock.patch('psutil._common.open', create=True) as m2: + assert psutil._pslinux.cpu_count_cores() is None + assert m1.called + assert m2.called + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemCPUFrequency(PsutilTestCase): + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + @pytest.mark.skipif( + AARCH64, reason="aarch64 does not always expose frequency" + ) + def test_emulate_use_second_file(self): + # https://github.com/giampaolo/psutil/issues/981 + def path_exists_mock(path): + if path.startswith("/sys/devices/system/cpu/cpufreq/policy"): + return False + else: + return orig_exists(path) + + orig_exists = os.path.exists + with mock.patch( + "os.path.exists", side_effect=path_exists_mock, create=True + ): + assert psutil.cpu_freq() + + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + @pytest.mark.skipif( + AARCH64, reason="aarch64 does not report mhz in /proc/cpuinfo" + ) + def test_emulate_use_cpuinfo(self): + # Emulate a case where /sys/devices/system/cpu/cpufreq* does not + # exist and /proc/cpuinfo is used instead. + def path_exists_mock(path): + if path.startswith('/sys/devices/system/cpu/'): + return False + else: + return os_path_exists(path) + + os_path_exists = os.path.exists + try: + with mock.patch("os.path.exists", side_effect=path_exists_mock): + reload_module(psutil._pslinux) + ret = psutil.cpu_freq() + assert ret, ret + assert ret.max == 0.0 + assert ret.min == 0.0 + for freq in psutil.cpu_freq(percpu=True): + assert freq.max == 0.0 + assert freq.min == 0.0 + finally: + reload_module(psutil._pslinux) + reload_module(psutil) + + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_data(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq') and name.startswith( + "/sys/devices/system/cpu/cpufreq/policy" + ): + return io.BytesIO(b"500000") + elif name.endswith('/scaling_min_freq') and name.startswith( + "/sys/devices/system/cpu/cpufreq/policy" + ): + return io.BytesIO(b"600000") + elif name.endswith('/scaling_max_freq') and name.startswith( + "/sys/devices/system/cpu/cpufreq/policy" + ): + return io.BytesIO(b"700000") + elif name == '/proc/cpuinfo': + return io.BytesIO(b"cpu MHz : 500") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('os.path.exists', return_value=True): + freq = psutil.cpu_freq() + assert freq.current == 500.0 + # when /proc/cpuinfo is used min and max frequencies are not + # available and are set to 0. + if freq.min != 0.0: + assert freq.min == 600.0 + if freq.max != 0.0: + assert freq.max == 700.0 + + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_multi_cpu(self): + def open_mock(name, *args, **kwargs): + n = name + if n.endswith('/scaling_cur_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy0" + ): + return io.BytesIO(b"100000") + elif n.endswith('/scaling_min_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy0" + ): + return io.BytesIO(b"200000") + elif n.endswith('/scaling_max_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy0" + ): + return io.BytesIO(b"300000") + elif n.endswith('/scaling_cur_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy1" + ): + return io.BytesIO(b"400000") + elif n.endswith('/scaling_min_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy1" + ): + return io.BytesIO(b"500000") + elif n.endswith('/scaling_max_freq') and n.startswith( + "/sys/devices/system/cpu/cpufreq/policy1" + ): + return io.BytesIO(b"600000") + elif name == '/proc/cpuinfo': + return io.BytesIO(b"cpu MHz : 100\ncpu MHz : 400") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('os.path.exists', return_value=True): + with mock.patch( + 'psutil._pslinux.cpu_count_logical', return_value=2 + ): + freq = psutil.cpu_freq(percpu=True) + assert freq[0].current == 100.0 + if freq[0].min != 0.0: + assert freq[0].min == 200.0 + if freq[0].max != 0.0: + assert freq[0].max == 300.0 + assert freq[1].current == 400.0 + if freq[1].min != 0.0: + assert freq[1].min == 500.0 + if freq[1].max != 0.0: + assert freq[1].max == 600.0 + + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + def test_emulate_no_scaling_cur_freq_file(self): + # See: https://github.com/giampaolo/psutil/issues/1071 + def open_mock(name, *args, **kwargs): + if name.endswith('/scaling_cur_freq'): + raise FileNotFoundError + if name.endswith('/cpuinfo_cur_freq'): + return io.BytesIO(b"200000") + elif name == '/proc/cpuinfo': + return io.BytesIO(b"cpu MHz : 200") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('os.path.exists', return_value=True): + with mock.patch( + 'psutil._pslinux.cpu_count_logical', return_value=1 + ): + freq = psutil.cpu_freq() + assert freq.current == 200 + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemCPUStats(PsutilTestCase): + + # XXX: fails too often. + # def test_ctx_switches(self): + # vmstat_value = vmstat("context switches") + # psutil_value = psutil.cpu_stats().ctx_switches + # self.assertAlmostEqual(vmstat_value, psutil_value, delta=500) + + def test_interrupts(self): + vmstat_value = vmstat("interrupts") + psutil_value = psutil.cpu_stats().interrupts + assert abs(vmstat_value - psutil_value) < 500 + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestLoadAvg(PsutilTestCase): + @pytest.mark.skipif(not HAS_GETLOADAVG, reason="not supported") + def test_getloadavg(self): + psutil_value = psutil.getloadavg() + with open("/proc/loadavg") as f: + proc_value = f.read().split() + + assert abs(float(proc_value[0]) - psutil_value[0]) < 1 + assert abs(float(proc_value[1]) - psutil_value[1]) < 1 + assert abs(float(proc_value[2]) - psutil_value[2]) < 1 + + +# ===================================================================== +# --- system network +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemNetIfAddrs(PsutilTestCase): + def test_ips(self): + for name, addrs in psutil.net_if_addrs().items(): + for addr in addrs: + if addr.family == psutil.AF_LINK: + assert addr.address == get_mac_address(name) + elif addr.family == socket.AF_INET: + assert addr.address == get_ipv4_address(name) + assert addr.netmask == get_ipv4_netmask(name) + if addr.broadcast is not None: + assert addr.broadcast == get_ipv4_broadcast(name) + else: + assert get_ipv4_broadcast(name) == '0.0.0.0' + elif addr.family == socket.AF_INET6: + # IPv6 addresses can have a percent symbol at the end. + # E.g. these 2 are equivalent: + # "fe80::1ff:fe23:4567:890a" + # "fe80::1ff:fe23:4567:890a%eth0" + # That is the "zone id" portion, which usually is the name + # of the network interface. + address = addr.address.split('%')[0] + assert address in get_ipv6_addresses(name) + + # XXX - not reliable when having virtual NICs installed by Docker. + # @pytest.mark.skipif(not shutil.which("ip"), + # reason="'ip' utility not available") + # def test_net_if_names(self): + # out = sh("ip addr").strip() + # nics = [x for x in psutil.net_if_addrs().keys() if ':' not in x] + # found = 0 + # for line in out.split('\n'): + # line = line.strip() + # if re.search(r"^\d+:", line): + # found += 1 + # name = line.split(':')[1].strip() + # self.assertIn(name, nics) + # self.assertEqual(len(nics), found, msg="{}\n---\n{}".format( + # pprint.pformat(nics), out)) + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemNetIfStats(PsutilTestCase): + @pytest.mark.skipif( + not shutil.which("ifconfig"), reason="ifconfig utility not available" + ) + def test_against_ifconfig(self): + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + assert stats.isup == ('RUNNING' in out), out + assert stats.mtu == int( + re.findall(r'(?i)MTU[: ](\d+)', out)[0] + ) + + def test_mtu(self): + for name, stats in psutil.net_if_stats().items(): + with open(f"/sys/class/net/{name}/mtu") as f: + assert stats.mtu == int(f.read().strip()) + + @pytest.mark.skipif( + not shutil.which("ifconfig"), reason="ifconfig utility not available" + ) + def test_flags(self): + # first line looks like this: + # "eth0: flags=4163 mtu 1500" + matches_found = 0 + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + match = re.search(r"flags=(\d+)?<(.*?)>", out) + if match and len(match.groups()) >= 2: + matches_found += 1 + ifconfig_flags = set(match.group(2).lower().split(",")) + psutil_flags = set(stats.flags.split(",")) + assert ifconfig_flags == psutil_flags + else: + # ifconfig has a different output on CentOS 6 + # let's try that + match = re.search(r"(.*) MTU:(\d+) Metric:(\d+)", out) + if match and len(match.groups()) >= 3: + matches_found += 1 + ifconfig_flags = set(match.group(1).lower().split()) + psutil_flags = set(stats.flags.split(",")) + assert ifconfig_flags == psutil_flags + + if not matches_found: + raise self.fail("no matches were found") + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemNetIOCounters(PsutilTestCase): + @pytest.mark.skipif( + not shutil.which("ifconfig"), reason="ifconfig utility not available" + ) + @retry_on_failure() + def test_against_ifconfig(self): + def ifconfig(nic): + ret = {} + out = sh(f"ifconfig {nic}") + ret['packets_recv'] = int( + re.findall(r'RX packets[: ](\d+)', out)[0] + ) + ret['packets_sent'] = int( + re.findall(r'TX packets[: ](\d+)', out)[0] + ) + ret['errin'] = int(re.findall(r'errors[: ](\d+)', out)[0]) + ret['errout'] = int(re.findall(r'errors[: ](\d+)', out)[1]) + ret['dropin'] = int(re.findall(r'dropped[: ](\d+)', out)[0]) + ret['dropout'] = int(re.findall(r'dropped[: ](\d+)', out)[1]) + ret['bytes_recv'] = int( + re.findall(r'RX (?:packets \d+ +)?bytes[: ](\d+)', out)[0] + ) + ret['bytes_sent'] = int( + re.findall(r'TX (?:packets \d+ +)?bytes[: ](\d+)', out)[0] + ) + return ret + + nio = psutil.net_io_counters(pernic=True, nowrap=False) + for name, stats in nio.items(): + try: + ifconfig_ret = ifconfig(name) + except RuntimeError: + continue + assert ( + abs(stats.bytes_recv - ifconfig_ret['bytes_recv']) < 1024 * 10 + ) + assert ( + abs(stats.bytes_sent - ifconfig_ret['bytes_sent']) < 1024 * 10 + ) + assert ( + abs(stats.packets_recv - ifconfig_ret['packets_recv']) < 1024 + ) + assert ( + abs(stats.packets_sent - ifconfig_ret['packets_sent']) < 1024 + ) + assert abs(stats.errin - ifconfig_ret['errin']) < 10 + assert abs(stats.errout - ifconfig_ret['errout']) < 10 + assert abs(stats.dropin - ifconfig_ret['dropin']) < 10 + assert abs(stats.dropout - ifconfig_ret['dropout']) < 10 + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemNetConnections(PsutilTestCase): + @mock.patch('psutil._pslinux.socket.inet_ntop', side_effect=ValueError) + @mock.patch('psutil._pslinux.supports_ipv6', return_value=False) + def test_emulate_ipv6_unsupported(self, supports_ipv6, inet_ntop): + # see: https://github.com/giampaolo/psutil/issues/623 + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self.addCleanup(s.close) + s.bind(("::1", 0)) + except OSError: + pass + psutil.net_connections(kind='inet6') + + def test_emulate_unix(self): + content = textwrap.dedent("""\ + 0: 00000003 000 000 0001 03 462170 @/tmp/dbus-Qw2hMPIU3n + 0: 00000003 000 000 0001 03 35010 @/tmp/dbus-tB2X8h69BQ + 0: 00000003 000 000 0001 03 34424 @/tmp/dbus-cHy80Y8O + 000000000000000000000000000000000000000000000000000000 + """) + with mock_open_content({"/proc/net/unix": content}) as m: + psutil.net_connections(kind='unix') + assert m.called + + +# ===================================================================== +# --- system disks +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemDiskPartitions(PsutilTestCase): + @pytest.mark.skipif( + not hasattr(os, 'statvfs'), reason="os.statvfs() not available" + ) + @skip_on_not_implemented() + def test_against_df(self): + # test psutil.disk_usage() and psutil.disk_partitions() + # against "df -a" + def df(path): + out = sh(f'df -P -B 1 "{path}"').strip() + lines = out.split('\n') + lines.pop(0) + line = lines.pop(0) + dev, total, used, free = line.split()[:4] + if dev == 'none': + dev = '' + total, used, free = int(total), int(used), int(free) + return dev, total, used, free + + for part in psutil.disk_partitions(all=False): + usage = psutil.disk_usage(part.mountpoint) + _, total, used, free = df(part.mountpoint) + assert usage.total == total + assert abs(usage.free - free) < TOLERANCE_DISK_USAGE + assert abs(usage.used - used) < TOLERANCE_DISK_USAGE + + def test_zfs_fs(self): + # Test that ZFS partitions are returned. + with open("/proc/filesystems") as f: + data = f.read() + if 'zfs' in data: + for part in psutil.disk_partitions(): + if part.fstype == 'zfs': + return + + # No ZFS partitions on this system. Let's fake one. + fake_file = io.StringIO("nodev\tzfs\n") + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m1: + with mock.patch( + 'psutil._pslinux.cext.disk_partitions', + return_value=[('/dev/sdb3', '/', 'zfs', 'rw')], + ) as m2: + ret = psutil.disk_partitions() + assert m1.called + assert m2.called + assert ret + assert ret[0].fstype == 'zfs' + + def test_emulate_realpath_fail(self): + # See: https://github.com/giampaolo/psutil/issues/1307 + try: + with mock.patch( + 'os.path.realpath', return_value='/non/existent' + ) as m: + with pytest.raises(FileNotFoundError): + psutil.disk_partitions() + assert m.called + finally: + psutil.PROCFS_PATH = "/proc" + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSystemDiskIoCounters(PsutilTestCase): + def test_emulate_kernel_2_4(self): + # Tests /proc/diskstats parsing format for 2.4 kernels, see: + # https://github.com/giampaolo/psutil/issues/767 + content = " 3 0 1 hda 2 3 4 5 6 7 8 9 10 11 12" + with mock_open_content({'/proc/diskstats': content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=True + ): + ret = psutil.disk_io_counters(nowrap=False) + assert ret.read_count == 1 + assert ret.read_merged_count == 2 + assert ret.read_bytes == 3 * SECTOR_SIZE + assert ret.read_time == 4 + assert ret.write_count == 5 + assert ret.write_merged_count == 6 + assert ret.write_bytes == 7 * SECTOR_SIZE + assert ret.write_time == 8 + assert ret.busy_time == 10 + + def test_emulate_kernel_2_6_full(self): + # Tests /proc/diskstats parsing format for 2.6 kernels, + # lines reporting all metrics: + # https://github.com/giampaolo/psutil/issues/767 + content = " 3 0 hda 1 2 3 4 5 6 7 8 9 10 11" + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=True + ): + ret = psutil.disk_io_counters(nowrap=False) + assert ret.read_count == 1 + assert ret.read_merged_count == 2 + assert ret.read_bytes == 3 * SECTOR_SIZE + assert ret.read_time == 4 + assert ret.write_count == 5 + assert ret.write_merged_count == 6 + assert ret.write_bytes == 7 * SECTOR_SIZE + assert ret.write_time == 8 + assert ret.busy_time == 10 + + def test_emulate_kernel_2_6_limited(self): + # Tests /proc/diskstats parsing format for 2.6 kernels, + # where one line of /proc/partitions return a limited + # amount of metrics when it bumps into a partition + # (instead of a disk). See: + # https://github.com/giampaolo/psutil/issues/767 + with mock_open_content({"/proc/diskstats": " 3 1 hda 1 2 3 4"}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=True + ): + ret = psutil.disk_io_counters(nowrap=False) + assert ret.read_count == 1 + assert ret.read_bytes == 2 * SECTOR_SIZE + assert ret.write_count == 3 + assert ret.write_bytes == 4 * SECTOR_SIZE + + assert ret.read_merged_count == 0 + assert ret.read_time == 0 + assert ret.write_merged_count == 0 + assert ret.write_time == 0 + assert ret.busy_time == 0 + + def test_emulate_include_partitions(self): + # Make sure that when perdisk=True disk partitions are returned, + # see: + # https://github.com/giampaolo/psutil/pull/1313#issuecomment-408626842 + content = textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """) + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=False + ): + ret = psutil.disk_io_counters(perdisk=True, nowrap=False) + assert len(ret) == 2 + assert ret['nvme0n1'].read_count == 1 + assert ret['nvme0n1p1'].read_count == 1 + assert ret['nvme0n1'].write_count == 5 + assert ret['nvme0n1p1'].write_count == 5 + + def test_emulate_exclude_partitions(self): + # Make sure that when perdisk=False partitions (e.g. 'sda1', + # 'nvme0n1p1') are skipped and not included in the total count. + # https://github.com/giampaolo/psutil/pull/1313#issuecomment-408626842 + content = textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """) + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', return_value=False + ): + ret = psutil.disk_io_counters(perdisk=False, nowrap=False) + assert ret is None + + def is_storage_device(name): + return name == 'nvme0n1' + + content = textwrap.dedent("""\ + 3 0 nvme0n1 1 2 3 4 5 6 7 8 9 10 11 + 3 0 nvme0n1p1 1 2 3 4 5 6 7 8 9 10 11 + """) + with mock_open_content({"/proc/diskstats": content}): + with mock.patch( + 'psutil._pslinux.is_storage_device', + create=True, + side_effect=is_storage_device, + ): + ret = psutil.disk_io_counters(perdisk=False, nowrap=False) + assert ret.read_count == 1 + assert ret.write_count == 5 + + def test_emulate_use_sysfs(self): + def exists(path): + return path == '/proc/diskstats' + + wprocfs = psutil.disk_io_counters(perdisk=True) + with mock.patch( + 'psutil._pslinux.os.path.exists', create=True, side_effect=exists + ): + wsysfs = psutil.disk_io_counters(perdisk=True) + assert len(wprocfs) == len(wsysfs) + + def test_emulate_not_impl(self): + def exists(path): + return False + + with mock.patch( + 'psutil._pslinux.os.path.exists', create=True, side_effect=exists + ): + with pytest.raises(NotImplementedError): + psutil.disk_io_counters() + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestRootFsDeviceFinder(PsutilTestCase): + def setUp(self): + dev = os.stat("/").st_dev + self.major = os.major(dev) + self.minor = os.minor(dev) + + def test_call_methods(self): + finder = RootFsDeviceFinder() + if os.path.exists("/proc/partitions"): + finder.ask_proc_partitions() + else: + with pytest.raises(FileNotFoundError): + finder.ask_proc_partitions() + if os.path.exists(f"/sys/dev/block/{self.major}:{self.minor}/uevent"): + finder.ask_sys_dev_block() + else: + with pytest.raises(FileNotFoundError): + finder.ask_sys_dev_block() + finder.ask_sys_class_block() + + @pytest.mark.skipif(GITHUB_ACTIONS, reason="unsupported on GITHUB_ACTIONS") + def test_comparisons(self): + finder = RootFsDeviceFinder() + assert finder.find() is not None + + a = b = c = None + if os.path.exists("/proc/partitions"): + a = finder.ask_proc_partitions() + if os.path.exists(f"/sys/dev/block/{self.major}:{self.minor}/uevent"): + b = finder.ask_sys_class_block() + c = finder.ask_sys_dev_block() + + base = a or b or c + if base and a: + assert base == a + if base and b: + assert base == b + if base and c: + assert base == c + + @pytest.mark.skipif( + not shutil.which("findmnt"), reason="findmnt utility not available" + ) + @pytest.mark.skipif(GITHUB_ACTIONS, reason="unsupported on GITHUB_ACTIONS") + def test_against_findmnt(self): + psutil_value = RootFsDeviceFinder().find() + findmnt_value = sh("findmnt -o SOURCE -rn /") + assert psutil_value == findmnt_value + + def test_disk_partitions_mocked(self): + with mock.patch( + 'psutil._pslinux.cext.disk_partitions', + return_value=[('/dev/root', '/', 'ext4', 'rw')], + ) as m: + part = psutil.disk_partitions()[0] + assert m.called + if not GITHUB_ACTIONS: + assert part.device != "/dev/root" + assert part.device == RootFsDeviceFinder().find() + else: + assert part.device == "/dev/root" + + +# ===================================================================== +# --- misc +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestMisc(PsutilTestCase): + def test_boot_time(self): + vmstat_value = vmstat('boot time') + psutil_value = psutil.boot_time() + assert int(vmstat_value) == int(psutil_value) + + def test_no_procfs_on_import(self): + my_procfs = self.get_testfn() + os.mkdir(my_procfs) + + with open(os.path.join(my_procfs, 'stat'), 'w') as f: + f.write('cpu 0 0 0 0 0 0 0 0 0 0\n') + f.write('cpu0 0 0 0 0 0 0 0 0 0 0\n') + f.write('cpu1 0 0 0 0 0 0 0 0 0 0\n') + + try: + orig_open = open + + def open_mock(name, *args, **kwargs): + if name.startswith('/proc'): + raise FileNotFoundError + return orig_open(name, *args, **kwargs) + + with mock.patch("builtins.open", side_effect=open_mock): + reload_module(psutil) + + with pytest.raises(OSError): + psutil.cpu_times() + with pytest.raises(OSError): + psutil.cpu_times(percpu=True) + with pytest.raises(OSError): + psutil.cpu_percent() + with pytest.raises(OSError): + psutil.cpu_percent(percpu=True) + with pytest.raises(OSError): + psutil.cpu_times_percent() + with pytest.raises(OSError): + psutil.cpu_times_percent(percpu=True) + + psutil.PROCFS_PATH = my_procfs + + assert psutil.cpu_percent() == 0 + assert sum(psutil.cpu_times_percent()) == 0 + + # since we don't know the number of CPUs at import time, + # we awkwardly say there are none until the second call + per_cpu_percent = psutil.cpu_percent(percpu=True) + assert sum(per_cpu_percent) == 0 + + # ditto awkward length + per_cpu_times_percent = psutil.cpu_times_percent(percpu=True) + assert sum(map(sum, per_cpu_times_percent)) == 0 + + # much user, very busy + with open(os.path.join(my_procfs, 'stat'), 'w') as f: + f.write('cpu 1 0 0 0 0 0 0 0 0 0\n') + f.write('cpu0 1 0 0 0 0 0 0 0 0 0\n') + f.write('cpu1 1 0 0 0 0 0 0 0 0 0\n') + + assert psutil.cpu_percent() != 0 + assert sum(psutil.cpu_percent(percpu=True)) != 0 + assert sum(psutil.cpu_times_percent()) != 0 + assert ( + sum(map(sum, psutil.cpu_times_percent(percpu=True))) != 0 + ) + finally: + shutil.rmtree(my_procfs) + reload_module(psutil) + + assert psutil.PROCFS_PATH == '/proc' + + def test_cpu_steal_decrease(self): + # Test cumulative cpu stats decrease. We should ignore this. + # See issue #1210. + content = textwrap.dedent("""\ + cpu 0 0 0 0 0 0 0 1 0 0 + cpu0 0 0 0 0 0 0 0 1 0 0 + cpu1 0 0 0 0 0 0 0 1 0 0 + """).encode() + with mock_open_content({"/proc/stat": content}) as m: + # first call to "percent" functions should read the new stat file + # and compare to the "real" file read at import time - so the + # values are meaningless + psutil.cpu_percent() + assert m.called + psutil.cpu_percent(percpu=True) + psutil.cpu_times_percent() + psutil.cpu_times_percent(percpu=True) + + content = textwrap.dedent("""\ + cpu 1 0 0 0 0 0 0 0 0 0 + cpu0 1 0 0 0 0 0 0 0 0 0 + cpu1 1 0 0 0 0 0 0 0 0 0 + """).encode() + with mock_open_content({"/proc/stat": content}): + # Increase "user" while steal goes "backwards" to zero. + cpu_percent = psutil.cpu_percent() + assert m.called + cpu_percent_percpu = psutil.cpu_percent(percpu=True) + cpu_times_percent = psutil.cpu_times_percent() + cpu_times_percent_percpu = psutil.cpu_times_percent(percpu=True) + assert cpu_percent != 0 + assert sum(cpu_percent_percpu) != 0 + assert sum(cpu_times_percent) != 0 + assert sum(cpu_times_percent) != 100.0 + assert sum(map(sum, cpu_times_percent_percpu)) != 0 + assert sum(map(sum, cpu_times_percent_percpu)) != 100.0 + assert cpu_times_percent.steal == 0 + assert cpu_times_percent.user != 0 + + def test_boot_time_mocked(self): + with mock.patch('psutil._common.open', create=True) as m: + with pytest.raises(RuntimeError): + psutil._pslinux.boot_time() + assert m.called + + def test_users(self): + # Make sure the C extension converts ':0' and ':0.0' to + # 'localhost'. + for user in psutil.users(): + assert user.host not in {":0", ":0.0"} + + def test_procfs_path(self): + tdir = self.get_testfn() + os.mkdir(tdir) + try: + psutil.PROCFS_PATH = tdir + with pytest.raises(OSError): + psutil.virtual_memory() + with pytest.raises(OSError): + psutil.cpu_times() + with pytest.raises(OSError): + psutil.cpu_times(percpu=True) + with pytest.raises(OSError): + psutil.boot_time() + # self.assertRaises(OSError, psutil.pids) + with pytest.raises(OSError): + psutil.net_connections() + with pytest.raises(OSError): + psutil.net_io_counters() + with pytest.raises(OSError): + psutil.net_if_stats() + # self.assertRaises(OSError, psutil.disk_io_counters) + with pytest.raises(OSError): + psutil.disk_partitions() + with pytest.raises(psutil.NoSuchProcess): + psutil.Process() + finally: + psutil.PROCFS_PATH = "/proc" + + @retry_on_failure() + @pytest.mark.skipif(PYTEST_PARALLEL, reason="skip if pytest-parallel") + def test_issue_687(self): + # In case of thread ID: + # - pid_exists() is supposed to return False + # - Process(tid) is supposed to work + # - pids() should not return the TID + # See: https://github.com/giampaolo/psutil/issues/687 + with ThreadTask(): + p = psutil.Process() + threads = p.threads() + assert len(threads) == 2 + tid = sorted(threads, key=lambda x: x.id)[1].id + assert p.pid != tid + pt = psutil.Process(tid) + pt.as_dict() + assert tid not in psutil.pids() + + def test_pid_exists_no_proc_status(self): + # Internally pid_exists relies on /proc/{pid}/status. + # Emulate a case where this file is empty in which case + # psutil is supposed to fall back on using pids(). + with mock_open_content({"/proc/%s/status": ""}) as m: + assert psutil.pid_exists(os.getpid()) + assert m.called + + +# ===================================================================== +# --- sensors +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +@pytest.mark.skipif(not HAS_BATTERY, reason="no battery") +class TestSensorsBattery(PsutilTestCase): + @pytest.mark.skipif( + not shutil.which("acpi"), reason="acpi utility not available" + ) + def test_percent(self): + out = sh("acpi -b") + acpi_value = int(out.split(",")[1].strip().replace('%', '')) + psutil_value = psutil.sensors_battery().percent + assert abs(acpi_value - psutil_value) < 1 + + def test_emulate_power_plugged(self): + # Pretend the AC power cable is connected. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + return io.BytesIO(b"1") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is True + assert ( + psutil.sensors_battery().secsleft + == psutil.POWER_TIME_UNLIMITED + ) + assert m.called + + def test_emulate_power_plugged_2(self): + # Same as above but pretend /AC0/online does not exist in which + # case code relies on /status file. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + raise FileNotFoundError + if name.endswith("/status"): + return io.StringIO("charging") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is True + assert m.called + + def test_emulate_power_not_plugged(self): + # Pretend the AC power cable is not connected. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + return io.BytesIO(b"0") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is False + assert m.called + + def test_emulate_power_not_plugged_2(self): + # Same as above but pretend /AC0/online does not exist in which + # case code relies on /status file. + def open_mock(name, *args, **kwargs): + if name.endswith(('AC0/online', 'AC/online')): + raise FileNotFoundError + if name.endswith("/status"): + return io.StringIO("discharging") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is False + assert m.called + + def test_emulate_power_undetermined(self): + # Pretend we can't know whether the AC power cable not + # connected (assert fallback to False). + def open_mock(name, *args, **kwargs): + if name.startswith(( + '/sys/class/power_supply/AC0/online', + '/sys/class/power_supply/AC/online', + )): + raise FileNotFoundError + if name.startswith("/sys/class/power_supply/BAT0/status"): + return io.BytesIO(b"???") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock) as m: + assert psutil.sensors_battery().power_plugged is None + assert m.called + + def test_emulate_energy_full_0(self): + # Emulate a case where energy_full files returns 0. + with mock_open_content( + {"/sys/class/power_supply/BAT0/energy_full": b"0"} + ) as m: + assert psutil.sensors_battery().percent == 0 + assert m.called + + def test_emulate_energy_full_not_avail(self): + # Emulate a case where energy_full file does not exist. + # Expected fallback on /capacity. + with mock_open_exception( + "/sys/class/power_supply/BAT0/energy_full", + FileNotFoundError, + ): + with mock_open_exception( + "/sys/class/power_supply/BAT0/charge_full", + FileNotFoundError, + ): + with mock_open_content( + {"/sys/class/power_supply/BAT0/capacity": b"88"} + ): + assert psutil.sensors_battery().percent == 88 + + def test_emulate_no_power(self): + # Emulate a case where /AC0/online file nor /BAT0/status exist. + with mock_open_exception( + "/sys/class/power_supply/AC/online", FileNotFoundError + ): + with mock_open_exception( + "/sys/class/power_supply/AC0/online", FileNotFoundError + ): + with mock_open_exception( + "/sys/class/power_supply/BAT0/status", + FileNotFoundError, + ): + assert psutil.sensors_battery().power_plugged is None + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSensorsBatteryEmulated(PsutilTestCase): + def test_it(self): + def open_mock(name, *args, **kwargs): + if name.endswith("/energy_now"): + return io.StringIO("60000000") + elif name.endswith("/power_now"): + return io.StringIO("0") + elif name.endswith("/energy_full"): + return io.StringIO("60000001") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch('os.listdir', return_value=["BAT0"]) as mlistdir: + with mock.patch("builtins.open", side_effect=open_mock) as mopen: + assert psutil.sensors_battery() is not None + assert mlistdir.called + assert mopen.called + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSensorsTemperatures(PsutilTestCase): + def test_emulate_class_hwmon(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/name'): + return io.StringIO("name") + elif name.endswith('/temp1_label'): + return io.StringIO("label") + elif name.endswith('/temp1_input'): + return io.BytesIO(b"30000") + elif name.endswith('/temp1_max'): + return io.BytesIO(b"40000") + elif name.endswith('/temp1_crit'): + return io.BytesIO(b"50000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + # Test case with /sys/class/hwmon + with mock.patch( + 'glob.glob', return_value=['/sys/class/hwmon/hwmon0/temp1'] + ): + temp = psutil.sensors_temperatures()['name'][0] + assert temp.label == 'label' + assert temp.current == 30.0 + assert temp.high == 40.0 + assert temp.critical == 50.0 + + def test_emulate_class_thermal(self): + def open_mock(name, *args, **kwargs): + if name.endswith('0_temp'): + return io.BytesIO(b"50000") + elif name.endswith('temp'): + return io.BytesIO(b"30000") + elif name.endswith('0_type'): + return io.StringIO("critical") + elif name.endswith('type'): + return io.StringIO("name") + else: + return orig_open(name, *args, **kwargs) + + def glob_mock(path): + if path in { + '/sys/class/hwmon/hwmon*/temp*_*', + '/sys/class/hwmon/hwmon*/device/temp*_*', + }: + return [] + elif path == '/sys/class/thermal/thermal_zone*': + return ['/sys/class/thermal/thermal_zone0'] + elif path == '/sys/class/thermal/thermal_zone0/trip_point*': + return [ + '/sys/class/thermal/thermal_zone1/trip_point_0_type', + '/sys/class/thermal/thermal_zone1/trip_point_0_temp', + ] + return [] + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch('glob.glob', create=True, side_effect=glob_mock): + temp = psutil.sensors_temperatures()['name'][0] + assert temp.label == '' + assert temp.current == 30.0 + assert temp.high == 50.0 + assert temp.critical == 50.0 + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestSensorsFans(PsutilTestCase): + def test_emulate_data(self): + def open_mock(name, *args, **kwargs): + if name.endswith('/name'): + return io.StringIO("name") + elif name.endswith('/fan1_label'): + return io.StringIO("label") + elif name.endswith('/fan1_input'): + return io.StringIO("2000") + else: + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock): + with mock.patch( + 'glob.glob', return_value=['/sys/class/hwmon/hwmon2/fan1'] + ): + fan = psutil.sensors_fans()['name'][0] + assert fan.label == 'label' + assert fan.current == 2000 + + +# ===================================================================== +# --- test process +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestProcess(PsutilTestCase): + @retry_on_failure() + def test_parse_smaps_vs_memory_maps(self): + sproc = self.spawn_testproc() + uss, pss, swap = psutil._pslinux.Process(sproc.pid)._parse_smaps() + maps = psutil.Process(sproc.pid).memory_maps(grouped=False) + assert ( + abs(uss - sum(x.private_dirty + x.private_clean for x in maps)) + < 4096 + ) + assert abs(pss - sum(x.pss for x in maps)) < 4096 + assert abs(swap - sum(x.swap for x in maps)) < 4096 + + def test_parse_smaps_mocked(self): + # See: https://github.com/giampaolo/psutil/issues/1222 + content = textwrap.dedent("""\ + fffff0 r-xp 00000000 00:00 0 [vsyscall] + Size: 1 kB + Rss: 2 kB + Pss: 3 kB + Shared_Clean: 4 kB + Shared_Dirty: 5 kB + Private_Clean: 6 kB + Private_Dirty: 7 kB + Referenced: 8 kB + Anonymous: 9 kB + LazyFree: 10 kB + AnonHugePages: 11 kB + ShmemPmdMapped: 12 kB + Shared_Hugetlb: 13 kB + Private_Hugetlb: 14 kB + Swap: 15 kB + SwapPss: 16 kB + KernelPageSize: 17 kB + MMUPageSize: 18 kB + Locked: 19 kB + VmFlags: rd ex + """).encode() + with mock_open_content({f"/proc/{os.getpid()}/smaps": content}) as m: + p = psutil._pslinux.Process(os.getpid()) + uss, pss, swap = p._parse_smaps() + assert m.called + assert uss == (6 + 7 + 14) * 1024 + assert pss == 3 * 1024 + assert swap == 15 * 1024 + + # On PYPY file descriptors are not closed fast enough. + @pytest.mark.skipif(PYPY, reason="unreliable on PYPY") + def test_open_files_mode(self): + def get_test_file(fname): + p = psutil.Process() + giveup_at = time.time() + GLOBAL_TIMEOUT + while True: + for file in p.open_files(): + if file.path == os.path.abspath(fname): + return file + elif time.time() > giveup_at: + break + raise RuntimeError("timeout looking for test file") + + testfn = self.get_testfn() + with open(testfn, "w"): + assert get_test_file(testfn).mode == "w" + with open(testfn): + assert get_test_file(testfn).mode == "r" + with open(testfn, "a"): + assert get_test_file(testfn).mode == "a" + with open(testfn, "r+"): + assert get_test_file(testfn).mode == "r+" + with open(testfn, "w+"): + assert get_test_file(testfn).mode == "r+" + with open(testfn, "a+"): + assert get_test_file(testfn).mode == "a+" + + safe_rmpath(testfn) + with open(testfn, "x"): + assert get_test_file(testfn).mode == "w" + safe_rmpath(testfn) + with open(testfn, "x+"): + assert get_test_file(testfn).mode == "r+" + + def test_open_files_file_gone(self): + # simulates a file which gets deleted during open_files() + # execution + p = psutil.Process() + files = p.open_files() + with open(self.get_testfn(), 'w'): + # give the kernel some time to see the new file + call_until(lambda: len(p.open_files()) != len(files)) + with mock.patch( + 'psutil._pslinux.os.readlink', + side_effect=FileNotFoundError, + ) as m: + assert p.open_files() == [] + assert m.called + # also simulate the case where os.readlink() returns EINVAL + # in which case psutil is supposed to 'continue' + with mock.patch( + 'psutil._pslinux.os.readlink', + side_effect=OSError(errno.EINVAL, ""), + ) as m: + assert p.open_files() == [] + assert m.called + + def test_open_files_fd_gone(self): + # Simulate a case where /proc/{pid}/fdinfo/{fd} disappears + # while iterating through fds. + # https://travis-ci.org/giampaolo/psutil/jobs/225694530 + p = psutil.Process() + files = p.open_files() + with open(self.get_testfn(), 'w'): + # give the kernel some time to see the new file + call_until(lambda: len(p.open_files()) != len(files)) + with mock.patch( + "builtins.open", side_effect=FileNotFoundError + ) as m: + assert p.open_files() == [] + assert m.called + + def test_open_files_enametoolong(self): + # Simulate a case where /proc/{pid}/fd/{fd} symlink + # points to a file with full path longer than PATH_MAX, see: + # https://github.com/giampaolo/psutil/issues/1940 + p = psutil.Process() + files = p.open_files() + with open(self.get_testfn(), 'w'): + # give the kernel some time to see the new file + call_until(lambda: len(p.open_files()) != len(files)) + patch_point = 'psutil._pslinux.os.readlink' + with mock.patch( + patch_point, side_effect=OSError(errno.ENAMETOOLONG, "") + ) as m: + with mock.patch("psutil._pslinux.debug"): + assert p.open_files() == [] + assert m.called + + # --- mocked tests + + def test_terminal_mocked(self): + with mock.patch( + 'psutil._pslinux._psposix.get_terminal_map', return_value={} + ) as m: + assert psutil._pslinux.Process(os.getpid()).terminal() is None + assert m.called + + # TODO: re-enable this test. + # def test_num_ctx_switches_mocked(self): + # with mock.patch('psutil._common.open', create=True) as m: + # self.assertRaises( + # NotImplementedError, + # psutil._pslinux.Process(os.getpid()).num_ctx_switches) + # assert m.called + + def test_cmdline_mocked(self): + # see: https://github.com/giampaolo/psutil/issues/639 + p = psutil.Process() + fake_file = io.StringIO('foo\x00bar\x00') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar'] + assert m.called + fake_file = io.StringIO('foo\x00bar\x00\x00') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar', ''] + assert m.called + + def test_cmdline_spaces_mocked(self): + # see: https://github.com/giampaolo/psutil/issues/1179 + p = psutil.Process() + fake_file = io.StringIO('foo bar ') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar'] + assert m.called + fake_file = io.StringIO('foo bar ') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar', ''] + assert m.called + + def test_cmdline_mixed_separators(self): + # https://github.com/giampaolo/psutil/issues/ + # 1179#issuecomment-552984549 + p = psutil.Process() + fake_file = io.StringIO('foo\x20bar\x00') + with mock.patch( + 'psutil._common.open', return_value=fake_file, create=True + ) as m: + assert p.cmdline() == ['foo', 'bar'] + assert m.called + + def test_readlink_path_deleted_mocked(self): + with mock.patch( + 'psutil._pslinux.os.readlink', return_value='/home/foo (deleted)' + ): + assert psutil.Process().exe() == "/home/foo" + assert psutil.Process().cwd() == "/home/foo" + + def test_threads_mocked(self): + # Test the case where os.listdir() returns a file (thread) + # which no longer exists by the time we open() it (race + # condition). threads() is supposed to ignore that instead + # of raising NSP. + def open_mock_1(name, *args, **kwargs): + if name.startswith(f"/proc/{os.getpid()}/task"): + raise FileNotFoundError + return orig_open(name, *args, **kwargs) + + orig_open = open + with mock.patch("builtins.open", side_effect=open_mock_1) as m: + ret = psutil.Process().threads() + assert m.called + assert ret == [] + + # ...but if it bumps into something != ENOENT we want an + # exception. + def open_mock_2(name, *args, **kwargs): + if name.startswith(f"/proc/{os.getpid()}/task"): + raise PermissionError + return orig_open(name, *args, **kwargs) + + with mock.patch("builtins.open", side_effect=open_mock_2): + with pytest.raises(psutil.AccessDenied): + psutil.Process().threads() + + def test_exe_mocked(self): + with mock.patch( + 'psutil._pslinux.readlink', side_effect=FileNotFoundError + ) as m: + # de-activate guessing from cmdline() + with mock.patch( + 'psutil._pslinux.Process.cmdline', return_value=[] + ): + ret = psutil.Process().exe() + assert m.called + assert ret == "" + + def test_issue_1014(self): + # Emulates a case where smaps file does not exist. In this case + # wrap_exception decorator should not raise NoSuchProcess. + with mock_open_exception( + f"/proc/{os.getpid()}/smaps", FileNotFoundError + ) as m: + p = psutil.Process() + with pytest.raises(FileNotFoundError): + p.memory_maps() + assert m.called + + def test_issue_2418(self): + p = psutil.Process() + with mock_open_exception( + f"/proc/{os.getpid()}/statm", FileNotFoundError + ): + with mock.patch("os.path.exists", return_value=False): + with pytest.raises(psutil.NoSuchProcess): + p.memory_info() + + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit_zombie(self): + # Emulate a case where rlimit() raises ENOSYS, which may + # happen in case of zombie process: + # https://travis-ci.org/giampaolo/psutil/jobs/51368273 + with mock.patch( + "resource.prlimit", side_effect=OSError(errno.ENOSYS, "") + ) as m1: + with mock.patch( + "psutil._pslinux.Process._is_zombie", return_value=True + ) as m2: + p = psutil.Process() + p.name() + with pytest.raises(psutil.ZombieProcess) as cm: + p.rlimit(psutil.RLIMIT_NOFILE) + assert m1.called + assert m2.called + assert cm.value.pid == p.pid + assert cm.value.name == p.name() + + def test_stat_file_parsing(self): + args = [ + "0", # pid + "(cat)", # name + "Z", # status + "1", # ppid + "0", # pgrp + "0", # session + "0", # tty + "0", # tpgid + "0", # flags + "0", # minflt + "0", # cminflt + "0", # majflt + "0", # cmajflt + "2", # utime + "3", # stime + "4", # cutime + "5", # cstime + "0", # priority + "0", # nice + "0", # num_threads + "0", # itrealvalue + "6", # starttime + "0", # vsize + "0", # rss + "0", # rsslim + "0", # startcode + "0", # endcode + "0", # startstack + "0", # kstkesp + "0", # kstkeip + "0", # signal + "0", # blocked + "0", # sigignore + "0", # sigcatch + "0", # wchan + "0", # nswap + "0", # cnswap + "0", # exit_signal + "6", # processor + "0", # rt priority + "0", # policy + "7", # delayacct_blkio_ticks + ] + content = " ".join(args).encode() + with mock_open_content({f"/proc/{os.getpid()}/stat": content}): + p = psutil.Process() + assert p.name() == 'cat' + assert p.status() == psutil.STATUS_ZOMBIE + assert p.ppid() == 1 + assert p.create_time() == 6 / CLOCK_TICKS + psutil.boot_time() + cpu = p.cpu_times() + assert cpu.user == 2 / CLOCK_TICKS + assert cpu.system == 3 / CLOCK_TICKS + assert cpu.children_user == 4 / CLOCK_TICKS + assert cpu.children_system == 5 / CLOCK_TICKS + assert cpu.iowait == 7 / CLOCK_TICKS + assert p.cpu_num() == 6 + + def test_status_file_parsing(self): + content = textwrap.dedent("""\ + Uid:\t1000\t1001\t1002\t1003 + Gid:\t1004\t1005\t1006\t1007 + Threads:\t66 + Cpus_allowed:\tf + Cpus_allowed_list:\t0-7 + voluntary_ctxt_switches:\t12 + nonvoluntary_ctxt_switches:\t13""").encode() + with mock_open_content({f"/proc/{os.getpid()}/status": content}): + p = psutil.Process() + assert p.num_ctx_switches().voluntary == 12 + assert p.num_ctx_switches().involuntary == 13 + assert p.num_threads() == 66 + uids = p.uids() + assert uids.real == 1000 + assert uids.effective == 1001 + assert uids.saved == 1002 + gids = p.gids() + assert gids.real == 1004 + assert gids.effective == 1005 + assert gids.saved == 1006 + assert p._proc._get_eligible_cpus() == list(range(8)) + + def test_net_connections_enametoolong(self): + # Simulate a case where /proc/{pid}/fd/{fd} symlink points to + # a file with full path longer than PATH_MAX, see: + # https://github.com/giampaolo/psutil/issues/1940 + with mock.patch( + 'psutil._pslinux.os.readlink', + side_effect=OSError(errno.ENAMETOOLONG, ""), + ) as m: + p = psutil.Process() + with mock.patch("psutil._pslinux.debug"): + assert p.net_connections() == [] + assert m.called + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestProcessAgainstStatus(PsutilTestCase): + """/proc/pid/stat and /proc/pid/status have many values in common. + Whenever possible, psutil uses /proc/pid/stat (it's faster). + For all those cases we check that the value found in + /proc/pid/stat (by psutil) matches the one found in + /proc/pid/status. + """ + + @classmethod + def setUpClass(cls): + cls.proc = psutil.Process() + + def read_status_file(self, linestart): + with psutil._psplatform.open_text( + f"/proc/{self.proc.pid}/status" + ) as f: + for line in f: + line = line.strip() + if line.startswith(linestart): + value = line.partition('\t')[2] + try: + return int(value) + except ValueError: + return value + raise ValueError(f"can't find {linestart!r}") + + def test_name(self): + value = self.read_status_file("Name:") + assert self.proc.name() == value + + def test_status(self): + value = self.read_status_file("State:") + value = value[value.find('(') + 1 : value.rfind(')')] + value = value.replace(' ', '-') + assert self.proc.status() == value + + def test_ppid(self): + value = self.read_status_file("PPid:") + assert self.proc.ppid() == value + + def test_num_threads(self): + value = self.read_status_file("Threads:") + assert self.proc.num_threads() == value + + def test_uids(self): + value = self.read_status_file("Uid:") + value = tuple(map(int, value.split()[1:4])) + assert self.proc.uids() == value + + def test_gids(self): + value = self.read_status_file("Gid:") + value = tuple(map(int, value.split()[1:4])) + assert self.proc.gids() == value + + @retry_on_failure() + def test_num_ctx_switches(self): + value = self.read_status_file("voluntary_ctxt_switches:") + assert self.proc.num_ctx_switches().voluntary == value + value = self.read_status_file("nonvoluntary_ctxt_switches:") + assert self.proc.num_ctx_switches().involuntary == value + + def test_cpu_affinity(self): + value = self.read_status_file("Cpus_allowed_list:") + if '-' in str(value): + min_, max_ = map(int, value.split('-')) + assert self.proc.cpu_affinity() == list(range(min_, max_ + 1)) + + def test_cpu_affinity_eligible_cpus(self): + value = self.read_status_file("Cpus_allowed_list:") + with mock.patch("psutil._pslinux.per_cpu_times") as m: + self.proc._proc._get_eligible_cpus() + if '-' in str(value): + assert not m.called + else: + assert m.called + + +# ===================================================================== +# --- test utils +# ===================================================================== + + +@pytest.mark.skipif(not LINUX, reason="LINUX only") +class TestUtils(PsutilTestCase): + def test_readlink(self): + with mock.patch("os.readlink", return_value="foo (deleted)") as m: + assert psutil._psplatform.readlink("bar") == "foo" + assert m.called diff --git a/lib/python3.12/site-packages/psutil/tests/test_memleaks.py b/lib/python3.12/site-packages/psutil/tests/test_memleaks.py new file mode 100644 index 0000000000000000000000000000000000000000..7f78fae67cbce20c4ec31e3703406f5135860a8e --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_memleaks.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for detecting function memory leaks (typically the ones +implemented in C). It does so by calling a function many times and +checking whether process memory usage keeps increasing between +calls or over time. +Note that this may produce false positives (especially on Windows +for some reason). +PyPy appears to be completely unstable for this framework, probably +because of how its JIT handles memory, so tests are skipped. +""" + + +import functools +import os +import platform + +import psutil +import psutil._common +from psutil import LINUX +from psutil import MACOS +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil.tests import HAS_CPU_AFFINITY +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_IONICE +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_NET_IO_COUNTERS +from psutil.tests import HAS_PROC_CPU_NUM +from psutil.tests import HAS_PROC_IO_COUNTERS +from psutil.tests import HAS_RLIMIT +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import TestMemoryLeak +from psutil.tests import create_sockets +from psutil.tests import get_testfn +from psutil.tests import process_namespace +from psutil.tests import pytest +from psutil.tests import skip_on_access_denied +from psutil.tests import spawn_testproc +from psutil.tests import system_namespace +from psutil.tests import terminate + + +cext = psutil._psplatform.cext +thisproc = psutil.Process() +FEW_TIMES = 5 + + +def fewtimes_if_linux(): + """Decorator for those Linux functions which are implemented in pure + Python, and which we want to run faster. + """ + + def decorator(fun): + @functools.wraps(fun) + def wrapper(self, *args, **kwargs): + if LINUX: + before = self.__class__.times + try: + self.__class__.times = FEW_TIMES + return fun(self, *args, **kwargs) + finally: + self.__class__.times = before + else: + return fun(self, *args, **kwargs) + + return wrapper + + return decorator + + +# =================================================================== +# Process class +# =================================================================== + + +class TestProcessObjectLeaks(TestMemoryLeak): + """Test leaks of Process class methods.""" + + proc = thisproc + + def test_coverage(self): + ns = process_namespace(None) + ns.test_class_coverage(self, ns.getters + ns.setters) + + @fewtimes_if_linux() + def test_name(self): + self.execute(self.proc.name) + + @fewtimes_if_linux() + def test_cmdline(self): + self.execute(self.proc.cmdline) + + @fewtimes_if_linux() + def test_exe(self): + self.execute(self.proc.exe) + + @fewtimes_if_linux() + def test_ppid(self): + self.execute(self.proc.ppid) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + @fewtimes_if_linux() + def test_uids(self): + self.execute(self.proc.uids) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + @fewtimes_if_linux() + def test_gids(self): + self.execute(self.proc.gids) + + @fewtimes_if_linux() + def test_status(self): + self.execute(self.proc.status) + + def test_nice(self): + self.execute(self.proc.nice) + + def test_nice_set(self): + niceness = thisproc.nice() + self.execute(lambda: self.proc.nice(niceness)) + + @pytest.mark.skipif(not HAS_IONICE, reason="not supported") + def test_ionice(self): + self.execute(self.proc.ionice) + + @pytest.mark.skipif(not HAS_IONICE, reason="not supported") + def test_ionice_set(self): + if WINDOWS: + value = thisproc.ionice() + self.execute(lambda: self.proc.ionice(value)) + else: + self.execute(lambda: self.proc.ionice(psutil.IOPRIO_CLASS_NONE)) + fun = functools.partial(cext.proc_ioprio_set, os.getpid(), -1, 0) + self.execute_w_exc(OSError, fun) + + @pytest.mark.skipif(not HAS_PROC_IO_COUNTERS, reason="not supported") + @fewtimes_if_linux() + def test_io_counters(self): + self.execute(self.proc.io_counters) + + @pytest.mark.skipif(POSIX, reason="worthless on POSIX") + def test_username(self): + # always open 1 handle on Windows (only once) + psutil.Process().username() + self.execute(self.proc.username) + + @fewtimes_if_linux() + def test_create_time(self): + self.execute(self.proc.create_time) + + @fewtimes_if_linux() + @skip_on_access_denied(only_if=OPENBSD) + def test_num_threads(self): + self.execute(self.proc.num_threads) + + @pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") + def test_num_handles(self): + self.execute(self.proc.num_handles) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + @fewtimes_if_linux() + def test_num_fds(self): + self.execute(self.proc.num_fds) + + @fewtimes_if_linux() + def test_num_ctx_switches(self): + self.execute(self.proc.num_ctx_switches) + + @fewtimes_if_linux() + @skip_on_access_denied(only_if=OPENBSD) + def test_threads(self): + self.execute(self.proc.threads) + + @fewtimes_if_linux() + def test_cpu_times(self): + self.execute(self.proc.cpu_times) + + @fewtimes_if_linux() + @pytest.mark.skipif(not HAS_PROC_CPU_NUM, reason="not supported") + def test_cpu_num(self): + self.execute(self.proc.cpu_num) + + @fewtimes_if_linux() + def test_memory_info(self): + self.execute(self.proc.memory_info) + + @fewtimes_if_linux() + def test_memory_full_info(self): + self.execute(self.proc.memory_full_info) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + @fewtimes_if_linux() + def test_terminal(self): + self.execute(self.proc.terminal) + + def test_resume(self): + times = FEW_TIMES if POSIX else self.times + self.execute(self.proc.resume, times=times) + + @fewtimes_if_linux() + def test_cwd(self): + self.execute(self.proc.cwd) + + @pytest.mark.skipif(not HAS_CPU_AFFINITY, reason="not supported") + def test_cpu_affinity(self): + self.execute(self.proc.cpu_affinity) + + @pytest.mark.skipif(not HAS_CPU_AFFINITY, reason="not supported") + def test_cpu_affinity_set(self): + affinity = thisproc.cpu_affinity() + self.execute(lambda: self.proc.cpu_affinity(affinity)) + self.execute_w_exc(ValueError, lambda: self.proc.cpu_affinity([-1])) + + @fewtimes_if_linux() + def test_open_files(self): + with open(get_testfn(), 'w'): + self.execute(self.proc.open_files) + + @pytest.mark.skipif(not HAS_MEMORY_MAPS, reason="not supported") + @fewtimes_if_linux() + def test_memory_maps(self): + self.execute(self.proc.memory_maps) + + @pytest.mark.skipif(not LINUX, reason="LINUX only") + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit(self): + self.execute(lambda: self.proc.rlimit(psutil.RLIMIT_NOFILE)) + + @pytest.mark.skipif(not LINUX, reason="LINUX only") + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit_set(self): + limit = thisproc.rlimit(psutil.RLIMIT_NOFILE) + self.execute(lambda: self.proc.rlimit(psutil.RLIMIT_NOFILE, limit)) + self.execute_w_exc((OSError, ValueError), lambda: self.proc.rlimit(-1)) + + @fewtimes_if_linux() + # Windows implementation is based on a single system-wide + # function (tested later). + @pytest.mark.skipif(WINDOWS, reason="worthless on WINDOWS") + def test_net_connections(self): + # TODO: UNIX sockets are temporarily implemented by parsing + # 'pfiles' cmd output; we don't want that part of the code to + # be executed. + with create_sockets(): + kind = 'inet' if SUNOS else 'all' + self.execute(lambda: self.proc.net_connections(kind)) + + @pytest.mark.skipif(not HAS_ENVIRON, reason="not supported") + def test_environ(self): + self.execute(self.proc.environ) + + @pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") + def test_proc_info(self): + self.execute(lambda: cext.proc_info(os.getpid())) + + +class TestTerminatedProcessLeaks(TestProcessObjectLeaks): + """Repeat the tests above looking for leaks occurring when dealing + with terminated processes raising NoSuchProcess exception. + The C functions are still invoked but will follow different code + paths. We'll check those code paths. + """ + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.subp = spawn_testproc() + cls.proc = psutil.Process(cls.subp.pid) + cls.proc.kill() + cls.proc.wait() + + @classmethod + def tearDownClass(cls): + super().tearDownClass() + terminate(cls.subp) + + def call(self, fun): + try: + fun() + except psutil.NoSuchProcess: + pass + + if WINDOWS: + + def test_kill(self): + self.execute(self.proc.kill) + + def test_terminate(self): + self.execute(self.proc.terminate) + + def test_suspend(self): + self.execute(self.proc.suspend) + + def test_resume(self): + self.execute(self.proc.resume) + + def test_wait(self): + self.execute(self.proc.wait) + + def test_proc_info(self): + # test dual implementation + def call(): + try: + return cext.proc_info(self.proc.pid) + except ProcessLookupError: + pass + + self.execute(call) + + +@pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") +class TestProcessDualImplementation(TestMemoryLeak): + def test_cmdline_peb_true(self): + self.execute(lambda: cext.proc_cmdline(os.getpid(), use_peb=True)) + + def test_cmdline_peb_false(self): + self.execute(lambda: cext.proc_cmdline(os.getpid(), use_peb=False)) + + +# =================================================================== +# system APIs +# =================================================================== + + +class TestModuleFunctionsLeaks(TestMemoryLeak): + """Test leaks of psutil module functions.""" + + def test_coverage(self): + ns = system_namespace() + ns.test_class_coverage(self, ns.all) + + # --- cpu + + @fewtimes_if_linux() + def test_cpu_count(self): # logical + self.execute(lambda: psutil.cpu_count(logical=True)) + + @fewtimes_if_linux() + def test_cpu_count_cores(self): + self.execute(lambda: psutil.cpu_count(logical=False)) + + @fewtimes_if_linux() + def test_cpu_times(self): + self.execute(psutil.cpu_times) + + @fewtimes_if_linux() + def test_per_cpu_times(self): + self.execute(lambda: psutil.cpu_times(percpu=True)) + + @fewtimes_if_linux() + def test_cpu_stats(self): + self.execute(psutil.cpu_stats) + + @fewtimes_if_linux() + # TODO: remove this once 1892 is fixed + @pytest.mark.skipif( + MACOS and platform.machine() == 'arm64', reason="skipped due to #1892" + ) + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + def test_cpu_freq(self): + self.execute(psutil.cpu_freq) + + @pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") + def test_getloadavg(self): + psutil.getloadavg() + self.execute(psutil.getloadavg) + + # --- mem + + def test_virtual_memory(self): + self.execute(psutil.virtual_memory) + + # TODO: remove this skip when this gets fixed + @pytest.mark.skipif(SUNOS, reason="worthless on SUNOS (uses a subprocess)") + def test_swap_memory(self): + self.execute(psutil.swap_memory) + + def test_pid_exists(self): + times = FEW_TIMES if POSIX else self.times + self.execute(lambda: psutil.pid_exists(os.getpid()), times=times) + + # --- disk + + def test_disk_usage(self): + times = FEW_TIMES if POSIX else self.times + self.execute(lambda: psutil.disk_usage('.'), times=times) + + def test_disk_partitions(self): + self.execute(psutil.disk_partitions) + + @pytest.mark.skipif( + LINUX and not os.path.exists('/proc/diskstats'), + reason="/proc/diskstats not available on this Linux version", + ) + @fewtimes_if_linux() + def test_disk_io_counters(self): + self.execute(lambda: psutil.disk_io_counters(nowrap=False)) + + # --- proc + + @fewtimes_if_linux() + def test_pids(self): + self.execute(psutil.pids) + + # --- net + + @fewtimes_if_linux() + @pytest.mark.skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_net_io_counters(self): + self.execute(lambda: psutil.net_io_counters(nowrap=False)) + + @fewtimes_if_linux() + @pytest.mark.skipif(MACOS and os.getuid() != 0, reason="need root access") + def test_net_connections(self): + # always opens and handle on Windows() (once) + psutil.net_connections(kind='all') + with create_sockets(): + self.execute(lambda: psutil.net_connections(kind='all')) + + def test_net_if_addrs(self): + # Note: verified that on Windows this was a false positive. + tolerance = 80 * 1024 if WINDOWS else self.tolerance + self.execute(psutil.net_if_addrs, tolerance=tolerance) + + def test_net_if_stats(self): + self.execute(psutil.net_if_stats) + + # --- sensors + + @fewtimes_if_linux() + @pytest.mark.skipif(not HAS_SENSORS_BATTERY, reason="not supported") + def test_sensors_battery(self): + self.execute(psutil.sensors_battery) + + @fewtimes_if_linux() + @pytest.mark.skipif(not HAS_SENSORS_TEMPERATURES, reason="not supported") + def test_sensors_temperatures(self): + self.execute(psutil.sensors_temperatures) + + @fewtimes_if_linux() + @pytest.mark.skipif(not HAS_SENSORS_FANS, reason="not supported") + def test_sensors_fans(self): + self.execute(psutil.sensors_fans) + + # --- others + + @fewtimes_if_linux() + def test_boot_time(self): + self.execute(psutil.boot_time) + + def test_users(self): + self.execute(psutil.users) + + def test_set_debug(self): + self.execute(lambda: psutil._set_debug(False)) + + if WINDOWS: + + # --- win services + + def test_win_service_iter(self): + self.execute(cext.winservice_enumerate) + + def test_win_service_get(self): + pass + + def test_win_service_get_config(self): + name = next(psutil.win_service_iter()).name() + self.execute(lambda: cext.winservice_query_config(name)) + + def test_win_service_get_status(self): + name = next(psutil.win_service_iter()).name() + self.execute(lambda: cext.winservice_query_status(name)) + + def test_win_service_get_description(self): + name = next(psutil.win_service_iter()).name() + self.execute(lambda: cext.winservice_query_descr(name)) diff --git a/lib/python3.12/site-packages/psutil/tests/test_misc.py b/lib/python3.12/site-packages/psutil/tests/test_misc.py new file mode 100644 index 0000000000000000000000000000000000000000..c484264b9190adcf4d7651a88eab53f1c354e845 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_misc.py @@ -0,0 +1,873 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Miscellaneous tests.""" + +import collections +import contextlib +import io +import json +import os +import pickle +import socket +import sys +from unittest import mock + +import psutil +import psutil.tests +from psutil import WINDOWS +from psutil._common import bcat +from psutil._common import cat +from psutil._common import debug +from psutil._common import isfile_strict +from psutil._common import memoize +from psutil._common import memoize_when_activated +from psutil._common import parse_environ_block +from psutil._common import supports_ipv6 +from psutil._common import wrap_numbers +from psutil.tests import HAS_NET_IO_COUNTERS +from psutil.tests import PsutilTestCase +from psutil.tests import process_namespace +from psutil.tests import pytest +from psutil.tests import reload_module +from psutil.tests import system_namespace + + +# =================================================================== +# --- Test classes' repr(), str(), ... +# =================================================================== + + +class TestSpecialMethods(PsutilTestCase): + def test_check_pid_range(self): + with pytest.raises(OverflowError): + psutil._psplatform.cext.check_pid_range(2**128) + with pytest.raises(psutil.NoSuchProcess): + psutil.Process(2**128) + + def test_process__repr__(self, func=repr): + p = psutil.Process(self.spawn_testproc().pid) + r = func(p) + assert "psutil.Process" in r + assert f"pid={p.pid}" in r + assert f"name='{p.name()}'" in r.replace("name=u'", "name='") + assert "status=" in r + assert "exitcode=" not in r + p.terminate() + p.wait() + r = func(p) + assert "status='terminated'" in r + assert "exitcode=" in r + + with mock.patch.object( + psutil.Process, + "name", + side_effect=psutil.ZombieProcess(os.getpid()), + ): + p = psutil.Process() + r = func(p) + assert f"pid={p.pid}" in r + assert "status='zombie'" in r + assert "name=" not in r + with mock.patch.object( + psutil.Process, + "name", + side_effect=psutil.NoSuchProcess(os.getpid()), + ): + p = psutil.Process() + r = func(p) + assert f"pid={p.pid}" in r + assert "terminated" in r + assert "name=" not in r + with mock.patch.object( + psutil.Process, + "name", + side_effect=psutil.AccessDenied(os.getpid()), + ): + p = psutil.Process() + r = func(p) + assert f"pid={p.pid}" in r + assert "name=" not in r + + def test_process__str__(self): + self.test_process__repr__(func=str) + + def test_error__repr__(self): + assert repr(psutil.Error()) == "psutil.Error()" + + def test_error__str__(self): + assert str(psutil.Error()) == "" + + def test_no_such_process__repr__(self): + assert ( + repr(psutil.NoSuchProcess(321)) + == "psutil.NoSuchProcess(pid=321, msg='process no longer exists')" + ) + assert ( + repr(psutil.NoSuchProcess(321, name="name", msg="msg")) + == "psutil.NoSuchProcess(pid=321, name='name', msg='msg')" + ) + + def test_no_such_process__str__(self): + assert ( + str(psutil.NoSuchProcess(321)) + == "process no longer exists (pid=321)" + ) + assert ( + str(psutil.NoSuchProcess(321, name="name", msg="msg")) + == "msg (pid=321, name='name')" + ) + + def test_zombie_process__repr__(self): + assert ( + repr(psutil.ZombieProcess(321)) + == 'psutil.ZombieProcess(pid=321, msg="PID still ' + 'exists but it\'s a zombie")' + ) + assert ( + repr(psutil.ZombieProcess(321, name="name", ppid=320, msg="foo")) + == "psutil.ZombieProcess(pid=321, ppid=320, name='name'," + " msg='foo')" + ) + + def test_zombie_process__str__(self): + assert ( + str(psutil.ZombieProcess(321)) + == "PID still exists but it's a zombie (pid=321)" + ) + assert ( + str(psutil.ZombieProcess(321, name="name", ppid=320, msg="foo")) + == "foo (pid=321, ppid=320, name='name')" + ) + + def test_access_denied__repr__(self): + assert repr(psutil.AccessDenied(321)) == "psutil.AccessDenied(pid=321)" + assert ( + repr(psutil.AccessDenied(321, name="name", msg="msg")) + == "psutil.AccessDenied(pid=321, name='name', msg='msg')" + ) + + def test_access_denied__str__(self): + assert str(psutil.AccessDenied(321)) == "(pid=321)" + assert ( + str(psutil.AccessDenied(321, name="name", msg="msg")) + == "msg (pid=321, name='name')" + ) + + def test_timeout_expired__repr__(self): + assert ( + repr(psutil.TimeoutExpired(5)) + == "psutil.TimeoutExpired(seconds=5, msg='timeout after 5" + " seconds')" + ) + assert ( + repr(psutil.TimeoutExpired(5, pid=321, name="name")) + == "psutil.TimeoutExpired(pid=321, name='name', seconds=5, " + "msg='timeout after 5 seconds')" + ) + + def test_timeout_expired__str__(self): + assert str(psutil.TimeoutExpired(5)) == "timeout after 5 seconds" + assert ( + str(psutil.TimeoutExpired(5, pid=321, name="name")) + == "timeout after 5 seconds (pid=321, name='name')" + ) + + def test_process__eq__(self): + p1 = psutil.Process() + p2 = psutil.Process() + assert p1 == p2 + p2._ident = (0, 0) + assert p1 != p2 + assert p1 != 'foo' + + def test_process__hash__(self): + s = {psutil.Process(), psutil.Process()} + assert len(s) == 1 + + +# =================================================================== +# --- Misc, generic, corner cases +# =================================================================== + + +class TestMisc(PsutilTestCase): + def test__all__(self): + dir_psutil = dir(psutil) + for name in dir_psutil: + if name in { + 'debug', + 'tests', + 'test', + 'PermissionError', + 'ProcessLookupError', + }: + continue + if not name.startswith('_'): + try: + __import__(name) + except ImportError: + if name not in psutil.__all__: + fun = getattr(psutil, name) + if fun is None: + continue + if ( + fun.__doc__ is not None + and 'deprecated' not in fun.__doc__.lower() + ): + raise self.fail(f"{name!r} not in psutil.__all__") + + # Import 'star' will break if __all__ is inconsistent, see: + # https://github.com/giampaolo/psutil/issues/656 + # Can't do `from psutil import *` as it won't work + # so we simply iterate over __all__. + for name in psutil.__all__: + assert name in dir_psutil + + def test_version(self): + assert ( + '.'.join([str(x) for x in psutil.version_info]) + == psutil.__version__ + ) + + def test_process_as_dict_no_new_names(self): + # See https://github.com/giampaolo/psutil/issues/813 + p = psutil.Process() + p.foo = '1' + assert 'foo' not in p.as_dict() + + def test_serialization(self): + def check(ret): + json.loads(json.dumps(ret)) + + a = pickle.dumps(ret) + b = pickle.loads(a) + assert ret == b + + # --- process APIs + + proc = psutil.Process() + check(psutil.Process().as_dict()) + + ns = process_namespace(proc) + for fun, name in ns.iter(ns.getters, clear_cache=True): + with self.subTest(proc=proc, name=name): + try: + ret = fun() + except psutil.Error: + pass + else: + check(ret) + + # --- system APIs + + ns = system_namespace() + for fun, name in ns.iter(ns.getters): + if name in {"win_service_iter", "win_service_get"}: + continue + with self.subTest(name=name): + try: + ret = fun() + except psutil.AccessDenied: + pass + else: + check(ret) + + # --- exception classes + + b = pickle.loads( + pickle.dumps( + psutil.NoSuchProcess(pid=4567, name='name', msg='msg') + ) + ) + assert isinstance(b, psutil.NoSuchProcess) + assert b.pid == 4567 + assert b.name == 'name' + assert b.msg == 'msg' + + b = pickle.loads( + pickle.dumps( + psutil.ZombieProcess(pid=4567, name='name', ppid=42, msg='msg') + ) + ) + assert isinstance(b, psutil.ZombieProcess) + assert b.pid == 4567 + assert b.ppid == 42 + assert b.name == 'name' + assert b.msg == 'msg' + + b = pickle.loads( + pickle.dumps(psutil.AccessDenied(pid=123, name='name', msg='msg')) + ) + assert isinstance(b, psutil.AccessDenied) + assert b.pid == 123 + assert b.name == 'name' + assert b.msg == 'msg' + + b = pickle.loads( + pickle.dumps( + psutil.TimeoutExpired(seconds=33, pid=4567, name='name') + ) + ) + assert isinstance(b, psutil.TimeoutExpired) + assert b.seconds == 33 + assert b.pid == 4567 + assert b.name == 'name' + + def test_ad_on_process_creation(self): + # We are supposed to be able to instantiate Process also in case + # of zombie processes or access denied. + with mock.patch.object( + psutil.Process, '_get_ident', side_effect=psutil.AccessDenied + ) as meth: + psutil.Process() + assert meth.called + + with mock.patch.object( + psutil.Process, '_get_ident', side_effect=psutil.ZombieProcess(1) + ) as meth: + psutil.Process() + assert meth.called + + with mock.patch.object( + psutil.Process, '_get_ident', side_effect=ValueError + ) as meth: + with pytest.raises(ValueError): + psutil.Process() + assert meth.called + + with mock.patch.object( + psutil.Process, '_get_ident', side_effect=psutil.NoSuchProcess(1) + ) as meth: + with self.assertRaises(psutil.NoSuchProcess): + psutil.Process() + assert meth.called + + def test_sanity_version_check(self): + # see: https://github.com/giampaolo/psutil/issues/564 + with mock.patch( + "psutil._psplatform.cext.version", return_value="0.0.0" + ): + with pytest.raises(ImportError) as cm: + reload_module(psutil) + assert "version conflict" in str(cm.value).lower() + + +# =================================================================== +# --- psutil/_common.py utils +# =================================================================== + + +class TestMemoizeDecorator(PsutilTestCase): + def setUp(self): + self.calls = [] + + tearDown = setUp + + def run_against(self, obj, expected_retval=None): + # no args + for _ in range(2): + ret = obj() + assert self.calls == [((), {})] + if expected_retval is not None: + assert ret == expected_retval + # with args + for _ in range(2): + ret = obj(1) + assert self.calls == [((), {}), ((1,), {})] + if expected_retval is not None: + assert ret == expected_retval + # with args + kwargs + for _ in range(2): + ret = obj(1, bar=2) + assert self.calls == [((), {}), ((1,), {}), ((1,), {'bar': 2})] + if expected_retval is not None: + assert ret == expected_retval + # clear cache + assert len(self.calls) == 3 + obj.cache_clear() + ret = obj() + if expected_retval is not None: + assert ret == expected_retval + assert len(self.calls) == 4 + # docstring + assert obj.__doc__ == "My docstring." + + def test_function(self): + @memoize + def foo(*args, **kwargs): + """My docstring.""" + baseclass.calls.append((args, kwargs)) + return 22 + + baseclass = self + self.run_against(foo, expected_retval=22) + + def test_class(self): + @memoize + class Foo: + """My docstring.""" + + def __init__(self, *args, **kwargs): + baseclass.calls.append((args, kwargs)) + + def bar(self): + return 22 + + baseclass = self + self.run_against(Foo, expected_retval=None) + assert Foo().bar() == 22 + + def test_class_singleton(self): + # @memoize can be used against classes to create singletons + @memoize + class Bar: + def __init__(self, *args, **kwargs): + pass + + assert Bar() is Bar() + assert id(Bar()) == id(Bar()) + assert id(Bar(1)) == id(Bar(1)) + assert id(Bar(1, foo=3)) == id(Bar(1, foo=3)) + assert id(Bar(1)) != id(Bar(2)) + + def test_staticmethod(self): + class Foo: + @staticmethod + @memoize + def bar(*args, **kwargs): + """My docstring.""" + baseclass.calls.append((args, kwargs)) + return 22 + + baseclass = self + self.run_against(Foo().bar, expected_retval=22) + + def test_classmethod(self): + class Foo: + @classmethod + @memoize + def bar(cls, *args, **kwargs): + """My docstring.""" + baseclass.calls.append((args, kwargs)) + return 22 + + baseclass = self + self.run_against(Foo().bar, expected_retval=22) + + def test_original(self): + # This was the original test before I made it dynamic to test it + # against different types. Keeping it anyway. + @memoize + def foo(*args, **kwargs): + """Foo docstring.""" + calls.append(None) + return (args, kwargs) + + calls = [] + # no args + for _ in range(2): + ret = foo() + expected = ((), {}) + assert ret == expected + assert len(calls) == 1 + # with args + for _ in range(2): + ret = foo(1) + expected = ((1,), {}) + assert ret == expected + assert len(calls) == 2 + # with args + kwargs + for _ in range(2): + ret = foo(1, bar=2) + expected = ((1,), {'bar': 2}) + assert ret == expected + assert len(calls) == 3 + # clear cache + foo.cache_clear() + ret = foo() + expected = ((), {}) + assert ret == expected + assert len(calls) == 4 + # docstring + assert foo.__doc__ == "Foo docstring." + + +class TestCommonModule(PsutilTestCase): + def test_memoize_when_activated(self): + class Foo: + @memoize_when_activated + def foo(self): + calls.append(None) + + f = Foo() + calls = [] + f.foo() + f.foo() + assert len(calls) == 2 + + # activate + calls = [] + f.foo.cache_activate(f) + f.foo() + f.foo() + assert len(calls) == 1 + + # deactivate + calls = [] + f.foo.cache_deactivate(f) + f.foo() + f.foo() + assert len(calls) == 2 + + def test_parse_environ_block(self): + def k(s): + return s.upper() if WINDOWS else s + + assert parse_environ_block("a=1\0") == {k("a"): "1"} + assert parse_environ_block("a=1\0b=2\0\0") == { + k("a"): "1", + k("b"): "2", + } + assert parse_environ_block("a=1\0b=\0\0") == {k("a"): "1", k("b"): ""} + # ignore everything after \0\0 + assert parse_environ_block("a=1\0b=2\0\0c=3\0") == { + k("a"): "1", + k("b"): "2", + } + # ignore everything that is not an assignment + assert parse_environ_block("xxx\0a=1\0") == {k("a"): "1"} + assert parse_environ_block("a=1\0=b=2\0") == {k("a"): "1"} + # do not fail if the block is incomplete + assert parse_environ_block("a=1\0b=2") == {k("a"): "1"} + + def test_supports_ipv6(self): + self.addCleanup(supports_ipv6.cache_clear) + if supports_ipv6(): + with mock.patch('psutil._common.socket') as s: + s.has_ipv6 = False + supports_ipv6.cache_clear() + assert not supports_ipv6() + + supports_ipv6.cache_clear() + with mock.patch( + 'psutil._common.socket.socket', side_effect=OSError + ) as s: + assert not supports_ipv6() + assert s.called + + supports_ipv6.cache_clear() + with mock.patch( + 'psutil._common.socket.socket', side_effect=socket.gaierror + ) as s: + assert not supports_ipv6() + supports_ipv6.cache_clear() + assert s.called + + supports_ipv6.cache_clear() + with mock.patch( + 'psutil._common.socket.socket.bind', + side_effect=socket.gaierror, + ) as s: + assert not supports_ipv6() + supports_ipv6.cache_clear() + assert s.called + else: + with pytest.raises(OSError): + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + try: + sock.bind(("::1", 0)) + finally: + sock.close() + + def test_isfile_strict(self): + this_file = os.path.abspath(__file__) + assert isfile_strict(this_file) + assert not isfile_strict(os.path.dirname(this_file)) + with mock.patch('psutil._common.os.stat', side_effect=PermissionError): + with pytest.raises(OSError): + isfile_strict(this_file) + with mock.patch( + 'psutil._common.os.stat', side_effect=FileNotFoundError + ): + assert not isfile_strict(this_file) + with mock.patch('psutil._common.stat.S_ISREG', return_value=False): + assert not isfile_strict(this_file) + + def test_debug(self): + with mock.patch.object(psutil._common, "PSUTIL_DEBUG", True): + with contextlib.redirect_stderr(io.StringIO()) as f: + debug("hello") + sys.stderr.flush() + msg = f.getvalue() + assert msg.startswith("psutil-debug"), msg + assert "hello" in msg + assert __file__.replace('.pyc', '.py') in msg + + # supposed to use repr(exc) + with mock.patch.object(psutil._common, "PSUTIL_DEBUG", True): + with contextlib.redirect_stderr(io.StringIO()) as f: + debug(ValueError("this is an error")) + msg = f.getvalue() + assert "ignoring ValueError" in msg + assert "'this is an error'" in msg + + # supposed to use str(exc), because of extra info about file name + with mock.patch.object(psutil._common, "PSUTIL_DEBUG", True): + with contextlib.redirect_stderr(io.StringIO()) as f: + exc = OSError(2, "no such file") + exc.filename = "/foo" + debug(exc) + msg = f.getvalue() + assert "no such file" in msg + assert "/foo" in msg + + def test_cat_bcat(self): + testfn = self.get_testfn() + with open(testfn, "w") as f: + f.write("foo") + assert cat(testfn) == "foo" + assert bcat(testfn) == b"foo" + with pytest.raises(FileNotFoundError): + cat(testfn + '-invalid') + with pytest.raises(FileNotFoundError): + bcat(testfn + '-invalid') + assert cat(testfn + '-invalid', fallback="bar") == "bar" + assert bcat(testfn + '-invalid', fallback="bar") == "bar" + + +# =================================================================== +# --- Tests for wrap_numbers() function. +# =================================================================== + + +nt = collections.namedtuple('foo', 'a b c') + + +class TestWrapNumbers(PsutilTestCase): + def setUp(self): + wrap_numbers.cache_clear() + + tearDown = setUp + + def test_first_call(self): + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + + def test_input_hasnt_changed(self): + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + assert wrap_numbers(input, 'disk_io') == input + + def test_increase_but_no_wrap(self): + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(10, 15, 20)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(20, 25, 30)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(20, 25, 30)} + assert wrap_numbers(input, 'disk_io') == input + + def test_wrap(self): + # let's say 100 is the threshold + input = {'disk1': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # first wrap restarts from 10 + input = {'disk1': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 110)} + # then it remains the same + input = {'disk1': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 110)} + # then it goes up + input = {'disk1': nt(100, 100, 90)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 190)} + # then it wraps again + input = {'disk1': nt(100, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 210)} + # and remains the same + input = {'disk1': nt(100, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(100, 100, 210)} + # now wrap another num + input = {'disk1': nt(50, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(150, 100, 210)} + # and again + input = {'disk1': nt(40, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(190, 100, 210)} + # keep it the same + input = {'disk1': nt(40, 100, 20)} + assert wrap_numbers(input, 'disk_io') == {'disk1': nt(190, 100, 210)} + + def test_changing_keys(self): + # Emulate a case where the second call to disk_io() + # (or whatever) provides a new disk, then the new disk + # disappears on the third call. + input = {'disk1': nt(5, 5, 5)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(5, 5, 5), 'disk2': nt(7, 7, 7)} + assert wrap_numbers(input, 'disk_io') == input + input = {'disk1': nt(8, 8, 8)} + assert wrap_numbers(input, 'disk_io') == input + + def test_changing_keys_w_wrap(self): + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # disk 2 wraps + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == { + 'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 110), + } + # disk 2 disappears + input = {'disk1': nt(50, 50, 50)} + assert wrap_numbers(input, 'disk_io') == input + + # then it appears again; the old wrap is supposed to be + # gone. + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # remains the same + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 100)} + assert wrap_numbers(input, 'disk_io') == input + # and then wraps again + input = {'disk1': nt(50, 50, 50), 'disk2': nt(100, 100, 10)} + assert wrap_numbers(input, 'disk_io') == { + 'disk1': nt(50, 50, 50), + 'disk2': nt(100, 100, 110), + } + + def test_real_data(self): + d = { + 'nvme0n1': (300, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348), + } + assert wrap_numbers(d, 'disk_io') == d + assert wrap_numbers(d, 'disk_io') == d + # decrease this ↓ + d = { + 'nvme0n1': (100, 508, 640, 1571, 5970, 1987, 2049, 451751, 47048), + 'nvme0n1p1': (1171, 2, 5600256, 1024, 516, 0, 0, 0, 8), + 'nvme0n1p2': (54, 54, 2396160, 5165056, 4, 24, 30, 1207, 28), + 'nvme0n1p3': (2389, 4539, 5154, 150, 4828, 1844, 2019, 398, 348), + } + out = wrap_numbers(d, 'disk_io') + assert out['nvme0n1'][0] == 400 + + # --- cache tests + + def test_cache_first_call(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == {'disk_io': {}} + assert cache[2] == {'disk_io': {}} + + def test_cache_call_twice(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + input = {'disk1': nt(10, 10, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 0} + } + assert cache[2] == {'disk_io': {}} + + def test_cache_wrap(self): + # let's say 100 is the threshold + input = {'disk1': nt(100, 100, 100)} + wrap_numbers(input, 'disk_io') + + # first wrap restarts from 10 + input = {'disk1': nt(100, 100, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 100} + } + assert cache[2] == {'disk_io': {'disk1': {('disk1', 2)}}} + + def check_cache_info(): + cache = wrap_numbers.cache_info() + assert cache[1] == { + 'disk_io': { + ('disk1', 0): 0, + ('disk1', 1): 0, + ('disk1', 2): 100, + } + } + assert cache[2] == {'disk_io': {'disk1': {('disk1', 2)}}} + + # then it remains the same + input = {'disk1': nt(100, 100, 10)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + check_cache_info() + + # then it goes up + input = {'disk1': nt(100, 100, 90)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + check_cache_info() + + # then it wraps again + input = {'disk1': nt(100, 100, 20)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 190} + } + assert cache[2] == {'disk_io': {'disk1': {('disk1', 2)}}} + + def test_cache_changing_keys(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + input = {'disk1': nt(5, 5, 5), 'disk2': nt(7, 7, 7)} + wrap_numbers(input, 'disk_io') + cache = wrap_numbers.cache_info() + assert cache[0] == {'disk_io': input} + assert cache[1] == { + 'disk_io': {('disk1', 0): 0, ('disk1', 1): 0, ('disk1', 2): 0} + } + assert cache[2] == {'disk_io': {}} + + def test_cache_clear(self): + input = {'disk1': nt(5, 5, 5)} + wrap_numbers(input, 'disk_io') + wrap_numbers(input, 'disk_io') + wrap_numbers.cache_clear('disk_io') + assert wrap_numbers.cache_info() == ({}, {}, {}) + wrap_numbers.cache_clear('disk_io') + wrap_numbers.cache_clear('?!?') + + @pytest.mark.skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_cache_clear_public_apis(self): + if not psutil.disk_io_counters() or not psutil.net_io_counters(): + raise pytest.skip("no disks or NICs available") + psutil.disk_io_counters() + psutil.net_io_counters() + caches = wrap_numbers.cache_info() + for cache in caches: + assert 'psutil.disk_io_counters' in cache + assert 'psutil.net_io_counters' in cache + + psutil.disk_io_counters.cache_clear() + caches = wrap_numbers.cache_info() + for cache in caches: + assert 'psutil.net_io_counters' in cache + assert 'psutil.disk_io_counters' not in cache + + psutil.net_io_counters.cache_clear() + caches = wrap_numbers.cache_info() + assert caches == ({}, {}, {}) diff --git a/lib/python3.12/site-packages/psutil/tests/test_osx.py b/lib/python3.12/site-packages/psutil/tests/test_osx.py new file mode 100644 index 0000000000000000000000000000000000000000..050418c5f9f107243cef4403d05cc27f9406b339 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_osx.py @@ -0,0 +1,197 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""macOS specific tests.""" + +import platform +import re +import time + +import psutil +from psutil import MACOS +from psutil import POSIX +from psutil.tests import CI_TESTING +from psutil.tests import HAS_BATTERY +from psutil.tests import TOLERANCE_DISK_USAGE +from psutil.tests import TOLERANCE_SYS_MEM +from psutil.tests import PsutilTestCase +from psutil.tests import pytest +from psutil.tests import retry_on_failure +from psutil.tests import sh +from psutil.tests import spawn_testproc +from psutil.tests import terminate + + +if POSIX: + from psutil._psutil_posix import getpagesize + + +def sysctl(cmdline): + """Expects a sysctl command with an argument and parse the result + returning only the value of interest. + """ + out = sh(cmdline) + result = out.split()[1] + try: + return int(result) + except ValueError: + return result + + +def vm_stat(field): + """Wrapper around 'vm_stat' cmdline utility.""" + out = sh('vm_stat') + for line in out.split('\n'): + if field in line: + break + else: + raise ValueError("line not found") + return int(re.search(r'\d+', line).group(0)) * getpagesize() + + +@pytest.mark.skipif(not MACOS, reason="MACOS only") +class TestProcess(PsutilTestCase): + @classmethod + def setUpClass(cls): + cls.pid = spawn_testproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + def test_process_create_time(self): + output = sh(f"ps -o lstart -p {self.pid}") + start_ps = output.replace('STARTED', '').strip() + hhmmss = start_ps.split(' ')[-2] + year = start_ps.split(' ')[-1] + start_psutil = psutil.Process(self.pid).create_time() + assert hhmmss == time.strftime( + "%H:%M:%S", time.localtime(start_psutil) + ) + assert year == time.strftime("%Y", time.localtime(start_psutil)) + + +@pytest.mark.skipif(not MACOS, reason="MACOS only") +class TestSystemAPIs(PsutilTestCase): + + # --- disk + + @retry_on_failure() + def test_disks(self): + # test psutil.disk_usage() and psutil.disk_partitions() + # against "df -a" + def df(path): + out = sh(f'df -k "{path}"').strip() + lines = out.split('\n') + lines.pop(0) + line = lines.pop(0) + dev, total, used, free = line.split()[:4] + if dev == 'none': + dev = '' + total = int(total) * 1024 + used = int(used) * 1024 + free = int(free) * 1024 + return dev, total, used, free + + for part in psutil.disk_partitions(all=False): + usage = psutil.disk_usage(part.mountpoint) + dev, total, used, free = df(part.mountpoint) + assert part.device == dev + assert usage.total == total + assert abs(usage.free - free) < TOLERANCE_DISK_USAGE + assert abs(usage.used - used) < TOLERANCE_DISK_USAGE + + # --- cpu + + def test_cpu_count_logical(self): + num = sysctl("sysctl hw.logicalcpu") + assert num == psutil.cpu_count(logical=True) + + def test_cpu_count_cores(self): + num = sysctl("sysctl hw.physicalcpu") + assert num == psutil.cpu_count(logical=False) + + # TODO: remove this once 1892 is fixed + @pytest.mark.skipif( + MACOS and platform.machine() == 'arm64', reason="skipped due to #1892" + ) + def test_cpu_freq(self): + freq = psutil.cpu_freq() + assert freq.current * 1000 * 1000 == sysctl("sysctl hw.cpufrequency") + assert freq.min * 1000 * 1000 == sysctl("sysctl hw.cpufrequency_min") + assert freq.max * 1000 * 1000 == sysctl("sysctl hw.cpufrequency_max") + + # --- virtual mem + + def test_vmem_total(self): + sysctl_hwphymem = sysctl('sysctl hw.memsize') + assert sysctl_hwphymem == psutil.virtual_memory().total + + @pytest.mark.skipif( + CI_TESTING and MACOS and platform.machine() == 'arm64', + reason="skipped on MACOS + ARM64 + CI_TESTING", + ) + @retry_on_failure() + def test_vmem_free(self): + vmstat_val = vm_stat("free") + psutil_val = psutil.virtual_memory().free + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_active(self): + vmstat_val = vm_stat("active") + psutil_val = psutil.virtual_memory().active + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_inactive(self): + vmstat_val = vm_stat("inactive") + psutil_val = psutil.virtual_memory().inactive + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_vmem_wired(self): + vmstat_val = vm_stat("wired") + psutil_val = psutil.virtual_memory().wired + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + # --- swap mem + + @retry_on_failure() + def test_swapmem_sin(self): + vmstat_val = vm_stat("Pageins") + psutil_val = psutil.swap_memory().sin + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + @retry_on_failure() + def test_swapmem_sout(self): + vmstat_val = vm_stat("Pageout") + psutil_val = psutil.swap_memory().sout + assert abs(psutil_val - vmstat_val) < TOLERANCE_SYS_MEM + + # --- network + + def test_net_if_stats(self): + for name, stats in psutil.net_if_stats().items(): + try: + out = sh(f"ifconfig {name}") + except RuntimeError: + pass + else: + assert stats.isup == ('RUNNING' in out), out + assert stats.mtu == int(re.findall(r'mtu (\d+)', out)[0]) + + # --- sensors_battery + + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_sensors_battery(self): + out = sh("pmset -g batt") + percent = re.search(r"(\d+)%", out).group(1) + drawing_from = re.search(r"Now drawing from '([^']+)'", out).group(1) + power_plugged = drawing_from == "AC Power" + psutil_result = psutil.sensors_battery() + assert psutil_result.power_plugged == power_plugged + assert psutil_result.percent == int(percent) diff --git a/lib/python3.12/site-packages/psutil/tests/test_posix.py b/lib/python3.12/site-packages/psutil/tests/test_posix.py new file mode 100644 index 0000000000000000000000000000000000000000..a7844929e767638086da0ebd85c27ab393f5c7ba --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_posix.py @@ -0,0 +1,488 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""POSIX specific tests.""" + +import datetime +import errno +import os +import re +import shutil +import subprocess +import time +from unittest import mock + +import psutil +from psutil import AIX +from psutil import BSD +from psutil import LINUX +from psutil import MACOS +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil.tests import AARCH64 +from psutil.tests import HAS_NET_IO_COUNTERS +from psutil.tests import PYTHON_EXE +from psutil.tests import PsutilTestCase +from psutil.tests import pytest +from psutil.tests import retry_on_failure +from psutil.tests import sh +from psutil.tests import skip_on_access_denied +from psutil.tests import spawn_testproc +from psutil.tests import terminate + + +if POSIX: + import mmap + import resource + + from psutil._psutil_posix import getpagesize + + +def ps(fmt, pid=None): + """Wrapper for calling the ps command with a little bit of cross-platform + support for a narrow range of features. + """ + + cmd = ['ps'] + + if LINUX: + cmd.append('--no-headers') + + if pid is not None: + cmd.extend(['-p', str(pid)]) + elif SUNOS or AIX: + cmd.append('-A') + else: + cmd.append('ax') + + if SUNOS: + fmt = fmt.replace("start", "stime") + + cmd.extend(['-o', fmt]) + + output = sh(cmd) + + output = output.splitlines() if LINUX else output.splitlines()[1:] + + all_output = [] + for line in output: + line = line.strip() + + try: + line = int(line) + except ValueError: + pass + + all_output.append(line) + + if pid is None: + return all_output + else: + return all_output[0] + + +# ps "-o" field names differ wildly between platforms. +# "comm" means "only executable name" but is not available on BSD platforms. +# "args" means "command with all its arguments", and is also not available +# on BSD platforms. +# "command" is like "args" on most platforms, but like "comm" on AIX, +# and not available on SUNOS. +# so for the executable name we can use "comm" on Solaris and split "command" +# on other platforms. +# to get the cmdline (with args) we have to use "args" on AIX and +# Solaris, and can use "command" on all others. + + +def ps_name(pid): + field = "command" + if SUNOS: + field = "comm" + command = ps(field, pid).split() + return command[0] + + +def ps_args(pid): + field = "command" + if AIX or SUNOS: + field = "args" + out = ps(field, pid) + # observed on BSD + Github CI: '/usr/local/bin/python3 -E -O (python3.9)' + out = re.sub(r"\(python.*?\)$", "", out) + return out.strip() + + +def ps_rss(pid): + field = "rss" + if AIX: + field = "rssize" + return ps(field, pid) + + +def ps_vsz(pid): + field = "vsz" + if AIX: + field = "vsize" + return ps(field, pid) + + +def df(device): + try: + out = sh(f"df -k {device}").strip() + except RuntimeError as err: + if "device busy" in str(err).lower(): + raise pytest.skip("df returned EBUSY") + raise + line = out.split('\n')[1] + fields = line.split() + sys_total = int(fields[1]) * 1024 + sys_used = int(fields[2]) * 1024 + sys_free = int(fields[3]) * 1024 + sys_percent = float(fields[4].replace('%', '')) + return (sys_total, sys_used, sys_free, sys_percent) + + +@pytest.mark.skipif(not POSIX, reason="POSIX only") +class TestProcess(PsutilTestCase): + """Compare psutil results against 'ps' command line utility (mainly).""" + + @classmethod + def setUpClass(cls): + cls.pid = spawn_testproc( + [PYTHON_EXE, "-E", "-O"], stdin=subprocess.PIPE + ).pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + def test_ppid(self): + ppid_ps = ps('ppid', self.pid) + ppid_psutil = psutil.Process(self.pid).ppid() + assert ppid_ps == ppid_psutil + + def test_uid(self): + uid_ps = ps('uid', self.pid) + uid_psutil = psutil.Process(self.pid).uids().real + assert uid_ps == uid_psutil + + def test_gid(self): + gid_ps = ps('rgid', self.pid) + gid_psutil = psutil.Process(self.pid).gids().real + assert gid_ps == gid_psutil + + def test_username(self): + username_ps = ps('user', self.pid) + username_psutil = psutil.Process(self.pid).username() + assert username_ps == username_psutil + + def test_username_no_resolution(self): + # Emulate a case where the system can't resolve the uid to + # a username in which case psutil is supposed to return + # the stringified uid. + p = psutil.Process() + with mock.patch("psutil.pwd.getpwuid", side_effect=KeyError) as fun: + assert p.username() == str(p.uids().real) + assert fun.called + + @skip_on_access_denied() + @retry_on_failure() + def test_rss_memory(self): + # give python interpreter some time to properly initialize + # so that the results are the same + time.sleep(0.1) + rss_ps = ps_rss(self.pid) + rss_psutil = psutil.Process(self.pid).memory_info()[0] / 1024 + assert rss_ps == rss_psutil + + @skip_on_access_denied() + @retry_on_failure() + def test_vsz_memory(self): + # give python interpreter some time to properly initialize + # so that the results are the same + time.sleep(0.1) + vsz_ps = ps_vsz(self.pid) + vsz_psutil = psutil.Process(self.pid).memory_info()[1] / 1024 + assert vsz_ps == vsz_psutil + + def test_name(self): + name_ps = ps_name(self.pid) + # remove path if there is any, from the command + name_ps = os.path.basename(name_ps).lower() + name_psutil = psutil.Process(self.pid).name().lower() + # ...because of how we calculate PYTHON_EXE; on MACOS this may + # be "pythonX.Y". + name_ps = re.sub(r"\d.\d", "", name_ps) + name_psutil = re.sub(r"\d.\d", "", name_psutil) + # ...may also be "python.X" + name_ps = re.sub(r"\d", "", name_ps) + name_psutil = re.sub(r"\d", "", name_psutil) + assert name_ps == name_psutil + + def test_name_long(self): + # On UNIX the kernel truncates the name to the first 15 + # characters. In such a case psutil tries to determine the + # full name from the cmdline. + name = "long-program-name" + cmdline = ["long-program-name-extended", "foo", "bar"] + with mock.patch("psutil._psplatform.Process.name", return_value=name): + with mock.patch( + "psutil._psplatform.Process.cmdline", return_value=cmdline + ): + p = psutil.Process() + assert p.name() == "long-program-name-extended" + + def test_name_long_cmdline_ad_exc(self): + # Same as above but emulates a case where cmdline() raises + # AccessDenied in which case psutil is supposed to return + # the truncated name instead of crashing. + name = "long-program-name" + with mock.patch("psutil._psplatform.Process.name", return_value=name): + with mock.patch( + "psutil._psplatform.Process.cmdline", + side_effect=psutil.AccessDenied(0, ""), + ): + p = psutil.Process() + assert p.name() == "long-program-name" + + def test_name_long_cmdline_nsp_exc(self): + # Same as above but emulates a case where cmdline() raises NSP + # which is supposed to propagate. + name = "long-program-name" + with mock.patch("psutil._psplatform.Process.name", return_value=name): + with mock.patch( + "psutil._psplatform.Process.cmdline", + side_effect=psutil.NoSuchProcess(0, ""), + ): + p = psutil.Process() + with pytest.raises(psutil.NoSuchProcess): + p.name() + + @pytest.mark.skipif(MACOS or BSD, reason="ps -o start not available") + def test_create_time(self): + time_ps = ps('start', self.pid) + time_psutil = psutil.Process(self.pid).create_time() + time_psutil_tstamp = datetime.datetime.fromtimestamp( + time_psutil + ).strftime("%H:%M:%S") + # sometimes ps shows the time rounded up instead of down, so we check + # for both possible values + round_time_psutil = round(time_psutil) + round_time_psutil_tstamp = datetime.datetime.fromtimestamp( + round_time_psutil + ).strftime("%H:%M:%S") + assert time_ps in {time_psutil_tstamp, round_time_psutil_tstamp} + + def test_exe(self): + ps_pathname = ps_name(self.pid) + psutil_pathname = psutil.Process(self.pid).exe() + try: + assert ps_pathname == psutil_pathname + except AssertionError: + # certain platforms such as BSD are more accurate returning: + # "/usr/local/bin/python3.7" + # ...instead of: + # "/usr/local/bin/python" + # We do not want to consider this difference in accuracy + # an error. + adjusted_ps_pathname = ps_pathname[: len(ps_pathname)] + assert ps_pathname == adjusted_ps_pathname + + # On macOS the official python installer exposes a python wrapper that + # executes a python executable hidden inside an application bundle inside + # the Python framework. + # There's a race condition between the ps call & the psutil call below + # depending on the completion of the execve call so let's retry on failure + @retry_on_failure() + def test_cmdline(self): + ps_cmdline = ps_args(self.pid) + psutil_cmdline = " ".join(psutil.Process(self.pid).cmdline()) + if AARCH64 and len(ps_cmdline) < len(psutil_cmdline): + assert psutil_cmdline.startswith(ps_cmdline) + else: + assert ps_cmdline == psutil_cmdline + + # On SUNOS "ps" reads niceness /proc/pid/psinfo which returns an + # incorrect value (20); the real deal is getpriority(2) which + # returns 0; psutil relies on it, see: + # https://github.com/giampaolo/psutil/issues/1082 + # AIX has the same issue + @pytest.mark.skipif(SUNOS, reason="not reliable on SUNOS") + @pytest.mark.skipif(AIX, reason="not reliable on AIX") + def test_nice(self): + ps_nice = ps('nice', self.pid) + psutil_nice = psutil.Process().nice() + assert ps_nice == psutil_nice + + +@pytest.mark.skipif(not POSIX, reason="POSIX only") +class TestSystemAPIs(PsutilTestCase): + """Test some system APIs.""" + + @retry_on_failure() + def test_pids(self): + # Note: this test might fail if the OS is starting/killing + # other processes in the meantime + pids_ps = sorted(ps("pid")) + pids_psutil = psutil.pids() + + # on MACOS and OPENBSD ps doesn't show pid 0 + if MACOS or (OPENBSD and 0 not in pids_ps): + pids_ps.insert(0, 0) + + # There will often be one more process in pids_ps for ps itself + if len(pids_ps) - len(pids_psutil) > 1: + difference = [x for x in pids_psutil if x not in pids_ps] + [ + x for x in pids_ps if x not in pids_psutil + ] + raise self.fail("difference: " + str(difference)) + + # for some reason ifconfig -a does not report all interfaces + # returned by psutil + @pytest.mark.skipif(SUNOS, reason="unreliable on SUNOS") + @pytest.mark.skipif(not shutil.which("ifconfig"), reason="no ifconfig cmd") + @pytest.mark.skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_nic_names(self): + output = sh("ifconfig -a") + for nic in psutil.net_io_counters(pernic=True): + for line in output.split(): + if line.startswith(nic): + break + else: + raise self.fail( + f"couldn't find {nic} nic in 'ifconfig -a'" + f" output\n{output}" + ) + + # @pytest.mark.skipif(CI_TESTING and not psutil.users(), + # reason="unreliable on CI") + @retry_on_failure() + def test_users(self): + out = sh("who -u") + if not out.strip(): + raise pytest.skip("no users on this system") + lines = out.split('\n') + users = [x.split()[0] for x in lines] + terminals = [x.split()[1] for x in lines] + assert len(users) == len(psutil.users()) + with self.subTest(psutil=psutil.users(), who=out): + for idx, u in enumerate(psutil.users()): + assert u.name == users[idx] + assert u.terminal == terminals[idx] + if u.pid is not None: # None on OpenBSD + psutil.Process(u.pid) + + @retry_on_failure() + def test_users_started(self): + out = sh("who -u") + if not out.strip(): + raise pytest.skip("no users on this system") + tstamp = None + # '2023-04-11 09:31' (Linux) + started = re.findall(r"\d\d\d\d-\d\d-\d\d \d\d:\d\d", out) + if started: + tstamp = "%Y-%m-%d %H:%M" + else: + # 'Apr 10 22:27' (macOS) + started = re.findall(r"[A-Z][a-z][a-z] \d\d \d\d:\d\d", out) + if started: + tstamp = "%b %d %H:%M" + else: + # 'Apr 10' + started = re.findall(r"[A-Z][a-z][a-z] \d\d", out) + if started: + tstamp = "%b %d" + else: + # 'apr 10' (sunOS) + started = re.findall(r"[a-z][a-z][a-z] \d\d", out) + if started: + tstamp = "%b %d" + started = [x.capitalize() for x in started] + + if not tstamp: + raise pytest.skip(f"cannot interpret tstamp in who output\n{out}") + + with self.subTest(psutil=psutil.users(), who=out): + for idx, u in enumerate(psutil.users()): + psutil_value = datetime.datetime.fromtimestamp( + u.started + ).strftime(tstamp) + assert psutil_value == started[idx] + + def test_pid_exists_let_raise(self): + # According to "man 2 kill" possible error values for kill + # are (EINVAL, EPERM, ESRCH). Test that any other errno + # results in an exception. + with mock.patch( + "psutil._psposix.os.kill", side_effect=OSError(errno.EBADF, "") + ) as m: + with pytest.raises(OSError): + psutil._psposix.pid_exists(os.getpid()) + assert m.called + + def test_os_waitpid_let_raise(self): + # os.waitpid() is supposed to catch EINTR and ECHILD only. + # Test that any other errno results in an exception. + with mock.patch( + "psutil._psposix.os.waitpid", side_effect=OSError(errno.EBADF, "") + ) as m: + with pytest.raises(OSError): + psutil._psposix.wait_pid(os.getpid()) + assert m.called + + def test_os_waitpid_eintr(self): + # os.waitpid() is supposed to "retry" on EINTR. + with mock.patch( + "psutil._psposix.os.waitpid", side_effect=OSError(errno.EINTR, "") + ) as m: + with pytest.raises(psutil._psposix.TimeoutExpired): + psutil._psposix.wait_pid(os.getpid(), timeout=0.01) + assert m.called + + def test_os_waitpid_bad_ret_status(self): + # Simulate os.waitpid() returning a bad status. + with mock.patch( + "psutil._psposix.os.waitpid", return_value=(1, -1) + ) as m: + with pytest.raises(ValueError): + psutil._psposix.wait_pid(os.getpid()) + assert m.called + + # AIX can return '-' in df output instead of numbers, e.g. for /proc + @pytest.mark.skipif(AIX, reason="unreliable on AIX") + @retry_on_failure() + def test_disk_usage(self): + tolerance = 4 * 1024 * 1024 # 4MB + for part in psutil.disk_partitions(all=False): + usage = psutil.disk_usage(part.mountpoint) + try: + sys_total, sys_used, sys_free, sys_percent = df(part.device) + except RuntimeError as err: + # see: + # https://travis-ci.org/giampaolo/psutil/jobs/138338464 + # https://travis-ci.org/giampaolo/psutil/jobs/138343361 + err = str(err).lower() + if ( + "no such file or directory" in err + or "raw devices not supported" in err + or "permission denied" in err + ): + continue + raise + else: + assert abs(usage.total - sys_total) < tolerance + assert abs(usage.used - sys_used) < tolerance + assert abs(usage.free - sys_free) < tolerance + assert abs(usage.percent - sys_percent) <= 1 + + +@pytest.mark.skipif(not POSIX, reason="POSIX only") +class TestMisc(PsutilTestCase): + def test_getpagesize(self): + pagesize = getpagesize() + assert pagesize > 0 + assert pagesize == resource.getpagesize() + assert pagesize == mmap.PAGESIZE diff --git a/lib/python3.12/site-packages/psutil/tests/test_process.py b/lib/python3.12/site-packages/psutil/tests/test_process.py new file mode 100644 index 0000000000000000000000000000000000000000..9ba1ba0e3bc7934a62011fcb89f880020ff984a0 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_process.py @@ -0,0 +1,1667 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for psutil.Process class.""" + +import collections +import contextlib +import errno +import getpass +import io +import itertools +import os +import signal +import socket +import stat +import string +import subprocess +import sys +import textwrap +import time +from unittest import mock + +import psutil +from psutil import AIX +from psutil import BSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import OSX +from psutil import POSIX +from psutil import WINDOWS +from psutil._common import open_text +from psutil.tests import CI_TESTING +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import GLOBAL_TIMEOUT +from psutil.tests import HAS_CPU_AFFINITY +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_IONICE +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_PROC_CPU_NUM +from psutil.tests import HAS_PROC_IO_COUNTERS +from psutil.tests import HAS_RLIMIT +from psutil.tests import HAS_THREADS +from psutil.tests import MACOS_11PLUS +from psutil.tests import PYPY +from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV +from psutil.tests import PsutilTestCase +from psutil.tests import ThreadTask +from psutil.tests import call_until +from psutil.tests import copyload_shared_lib +from psutil.tests import create_c_exe +from psutil.tests import create_py_exe +from psutil.tests import process_namespace +from psutil.tests import pytest +from psutil.tests import reap_children +from psutil.tests import retry_on_failure +from psutil.tests import sh +from psutil.tests import skip_on_access_denied +from psutil.tests import skip_on_not_implemented +from psutil.tests import wait_for_pid + + +# =================================================================== +# --- psutil.Process class tests +# =================================================================== + + +class TestProcess(PsutilTestCase): + """Tests for psutil.Process class.""" + + def spawn_psproc(self, *args, **kwargs): + sproc = self.spawn_testproc(*args, **kwargs) + try: + return psutil.Process(sproc.pid) + except psutil.NoSuchProcess: + self.assertPidGone(sproc.pid) + raise + + # --- + + def test_pid(self): + p = psutil.Process() + assert p.pid == os.getpid() + with pytest.raises(AttributeError): + p.pid = 33 + + def test_kill(self): + p = self.spawn_psproc() + p.kill() + code = p.wait() + if WINDOWS: + assert code == signal.SIGTERM + else: + assert code == -signal.SIGKILL + self.assertProcessGone(p) + + def test_terminate(self): + p = self.spawn_psproc() + p.terminate() + code = p.wait() + if WINDOWS: + assert code == signal.SIGTERM + else: + assert code == -signal.SIGTERM + self.assertProcessGone(p) + + def test_send_signal(self): + sig = signal.SIGKILL if POSIX else signal.SIGTERM + p = self.spawn_psproc() + p.send_signal(sig) + code = p.wait() + if WINDOWS: + assert code == sig + else: + assert code == -sig + self.assertProcessGone(p) + + @pytest.mark.skipif(not POSIX, reason="not POSIX") + def test_send_signal_mocked(self): + sig = signal.SIGTERM + p = self.spawn_psproc() + with mock.patch('psutil.os.kill', side_effect=ProcessLookupError): + with pytest.raises(psutil.NoSuchProcess): + p.send_signal(sig) + + p = self.spawn_psproc() + with mock.patch('psutil.os.kill', side_effect=PermissionError): + with pytest.raises(psutil.AccessDenied): + p.send_signal(sig) + + def test_wait_exited(self): + # Test waitpid() + WIFEXITED -> WEXITSTATUS. + # normal return, same as exit(0) + cmd = [PYTHON_EXE, "-c", "pass"] + p = self.spawn_psproc(cmd) + code = p.wait() + assert code == 0 + self.assertProcessGone(p) + # exit(1), implicit in case of error + cmd = [PYTHON_EXE, "-c", "1 / 0"] + p = self.spawn_psproc(cmd, stderr=subprocess.PIPE) + code = p.wait() + assert code == 1 + self.assertProcessGone(p) + # via sys.exit() + cmd = [PYTHON_EXE, "-c", "import sys; sys.exit(5);"] + p = self.spawn_psproc(cmd) + code = p.wait() + assert code == 5 + self.assertProcessGone(p) + # via os._exit() + cmd = [PYTHON_EXE, "-c", "import os; os._exit(5);"] + p = self.spawn_psproc(cmd) + code = p.wait() + assert code == 5 + self.assertProcessGone(p) + + @pytest.mark.skipif(NETBSD, reason="fails on NETBSD") + def test_wait_stopped(self): + p = self.spawn_psproc() + if POSIX: + # Test waitpid() + WIFSTOPPED and WIFCONTINUED. + # Note: if a process is stopped it ignores SIGTERM. + p.send_signal(signal.SIGSTOP) + with pytest.raises(psutil.TimeoutExpired): + p.wait(timeout=0.001) + p.send_signal(signal.SIGCONT) + with pytest.raises(psutil.TimeoutExpired): + p.wait(timeout=0.001) + p.send_signal(signal.SIGTERM) + assert p.wait() == -signal.SIGTERM + assert p.wait() == -signal.SIGTERM + else: + p.suspend() + with pytest.raises(psutil.TimeoutExpired): + p.wait(timeout=0.001) + p.resume() + with pytest.raises(psutil.TimeoutExpired): + p.wait(timeout=0.001) + p.terminate() + assert p.wait() == signal.SIGTERM + assert p.wait() == signal.SIGTERM + + def test_wait_non_children(self): + # Test wait() against a process which is not our direct + # child. + child, grandchild = self.spawn_children_pair() + with pytest.raises(psutil.TimeoutExpired): + child.wait(0.01) + with pytest.raises(psutil.TimeoutExpired): + grandchild.wait(0.01) + # We also terminate the direct child otherwise the + # grandchild will hang until the parent is gone. + child.terminate() + grandchild.terminate() + child_ret = child.wait() + grandchild_ret = grandchild.wait() + if POSIX: + assert child_ret == -signal.SIGTERM + # For processes which are not our children we're supposed + # to get None. + assert grandchild_ret is None + else: + assert child_ret == signal.SIGTERM + assert child_ret == signal.SIGTERM + + def test_wait_timeout(self): + p = self.spawn_psproc() + p.name() + with pytest.raises(psutil.TimeoutExpired): + p.wait(0.01) + with pytest.raises(psutil.TimeoutExpired): + p.wait(0) + with pytest.raises(ValueError): + p.wait(-1) + + def test_wait_timeout_nonblocking(self): + p = self.spawn_psproc() + with pytest.raises(psutil.TimeoutExpired): + p.wait(0) + p.kill() + stop_at = time.time() + GLOBAL_TIMEOUT + while time.time() < stop_at: + try: + code = p.wait(0) + break + except psutil.TimeoutExpired: + pass + else: + raise self.fail('timeout') + if POSIX: + assert code == -signal.SIGKILL + else: + assert code == signal.SIGTERM + self.assertProcessGone(p) + + def test_cpu_percent(self): + p = psutil.Process() + p.cpu_percent(interval=0.001) + p.cpu_percent(interval=0.001) + for _ in range(100): + percent = p.cpu_percent(interval=None) + assert isinstance(percent, float) + assert percent >= 0.0 + with pytest.raises(ValueError): + p.cpu_percent(interval=-1) + + def test_cpu_percent_numcpus_none(self): + # See: https://github.com/giampaolo/psutil/issues/1087 + with mock.patch('psutil.cpu_count', return_value=None) as m: + psutil.Process().cpu_percent() + assert m.called + + def test_cpu_times(self): + times = psutil.Process().cpu_times() + assert times.user >= 0.0, times + assert times.system >= 0.0, times + assert times.children_user >= 0.0, times + assert times.children_system >= 0.0, times + if LINUX: + assert times.iowait >= 0.0, times + # make sure returned values can be pretty printed with strftime + for name in times._fields: + time.strftime("%H:%M:%S", time.localtime(getattr(times, name))) + + def test_cpu_times_2(self): + def waste_cpu(): + stop_at = os.times().user + 0.2 + while os.times().user < stop_at: + for x in range(100000): + x **= 2 + + waste_cpu() + a = psutil.Process().cpu_times() + b = os.times() + self.assertAlmostEqual(a.user, b.user, delta=0.1) + self.assertAlmostEqual(a.system, b.system, delta=0.1) + + @pytest.mark.skipif(not HAS_PROC_CPU_NUM, reason="not supported") + def test_cpu_num(self): + p = psutil.Process() + num = p.cpu_num() + assert num >= 0 + if psutil.cpu_count() == 1: + assert num == 0 + assert p.cpu_num() in range(psutil.cpu_count()) + + def test_create_time(self): + p = self.spawn_psproc() + now = time.time() + create_time = p.create_time() + + # Use time.time() as base value to compare our result using a + # tolerance of +/- 1 second. + # It will fail if the difference between the values is > 2s. + difference = abs(create_time - now) + if difference > 2: + raise self.fail( + f"expected: {now}, found: {create_time}, difference:" + f" {difference}" + ) + + # make sure returned value can be pretty printed with strftime + time.strftime("%Y %m %d %H:%M:%S", time.localtime(p.create_time())) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_terminal(self): + terminal = psutil.Process().terminal() + if terminal is not None: + try: + tty = os.path.realpath(sh('tty')) + except RuntimeError: + # Note: happens if pytest is run without the `-s` opt. + raise pytest.skip("can't rely on `tty` CLI") + else: + assert terminal == tty + + @pytest.mark.skipif(not HAS_PROC_IO_COUNTERS, reason="not supported") + @skip_on_not_implemented(only_if=LINUX) + def test_io_counters(self): + p = psutil.Process() + # test reads + io1 = p.io_counters() + with open(PYTHON_EXE, 'rb') as f: + f.read() + io2 = p.io_counters() + if not BSD and not AIX: + assert io2.read_count > io1.read_count + assert io2.write_count == io1.write_count + if LINUX: + assert io2.read_chars > io1.read_chars + assert io2.write_chars == io1.write_chars + else: + assert io2.read_bytes >= io1.read_bytes + assert io2.write_bytes >= io1.write_bytes + + # test writes + io1 = p.io_counters() + with open(self.get_testfn(), 'wb') as f: + f.write(bytes("x" * 1000000, 'ascii')) + io2 = p.io_counters() + assert io2.write_count >= io1.write_count + assert io2.write_bytes >= io1.write_bytes + assert io2.read_count >= io1.read_count + assert io2.read_bytes >= io1.read_bytes + if LINUX: + assert io2.write_chars > io1.write_chars + assert io2.read_chars >= io1.read_chars + + # sanity check + for i in range(len(io2)): + if BSD and i >= 2: + # On BSD read_bytes and write_bytes are always set to -1. + continue + assert io2[i] >= 0 + assert io2[i] >= 0 + + @pytest.mark.skipif(not HAS_IONICE, reason="not supported") + @pytest.mark.skipif(not LINUX, reason="linux only") + def test_ionice_linux(self): + def cleanup(init): + ioclass, value = init + if ioclass == psutil.IOPRIO_CLASS_NONE: + value = 0 + p.ionice(ioclass, value) + + p = psutil.Process() + if not CI_TESTING: + assert p.ionice()[0] == psutil.IOPRIO_CLASS_NONE + assert psutil.IOPRIO_CLASS_NONE == 0 + assert psutil.IOPRIO_CLASS_RT == 1 # high + assert psutil.IOPRIO_CLASS_BE == 2 # normal + assert psutil.IOPRIO_CLASS_IDLE == 3 # low + init = p.ionice() + self.addCleanup(cleanup, init) + + # low + p.ionice(psutil.IOPRIO_CLASS_IDLE) + assert tuple(p.ionice()) == (psutil.IOPRIO_CLASS_IDLE, 0) + with pytest.raises(ValueError): # accepts no value + p.ionice(psutil.IOPRIO_CLASS_IDLE, value=7) + # normal + p.ionice(psutil.IOPRIO_CLASS_BE) + assert tuple(p.ionice()) == (psutil.IOPRIO_CLASS_BE, 0) + p.ionice(psutil.IOPRIO_CLASS_BE, value=7) + assert tuple(p.ionice()) == (psutil.IOPRIO_CLASS_BE, 7) + with pytest.raises(ValueError): + p.ionice(psutil.IOPRIO_CLASS_BE, value=8) + try: + p.ionice(psutil.IOPRIO_CLASS_RT, value=7) + except psutil.AccessDenied: + pass + # errs + with pytest.raises(ValueError, match="ioclass accepts no value"): + p.ionice(psutil.IOPRIO_CLASS_NONE, 1) + with pytest.raises(ValueError, match="ioclass accepts no value"): + p.ionice(psutil.IOPRIO_CLASS_IDLE, 1) + with pytest.raises( + ValueError, match="'ioclass' argument must be specified" + ): + p.ionice(value=1) + + @pytest.mark.skipif(not HAS_IONICE, reason="not supported") + @pytest.mark.skipif( + not WINDOWS, reason="not supported on this win version" + ) + def test_ionice_win(self): + p = psutil.Process() + if not CI_TESTING: + assert p.ionice() == psutil.IOPRIO_NORMAL + init = p.ionice() + self.addCleanup(p.ionice, init) + + # base + p.ionice(psutil.IOPRIO_VERYLOW) + assert p.ionice() == psutil.IOPRIO_VERYLOW + p.ionice(psutil.IOPRIO_LOW) + assert p.ionice() == psutil.IOPRIO_LOW + try: + p.ionice(psutil.IOPRIO_HIGH) + except psutil.AccessDenied: + pass + else: + assert p.ionice() == psutil.IOPRIO_HIGH + # errs + with pytest.raises( + TypeError, match="value argument not accepted on Windows" + ): + p.ionice(psutil.IOPRIO_NORMAL, value=1) + with pytest.raises(ValueError, match="is not a valid priority"): + p.ionice(psutil.IOPRIO_HIGH + 1) + + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit_get(self): + import resource + + p = psutil.Process(os.getpid()) + names = [x for x in dir(psutil) if x.startswith('RLIMIT')] + assert names, names + for name in names: + value = getattr(psutil, name) + assert value >= 0 + if name in dir(resource): + assert value == getattr(resource, name) + # XXX - On PyPy RLIMIT_INFINITY returned by + # resource.getrlimit() is reported as a very big long + # number instead of -1. It looks like a bug with PyPy. + if PYPY: + continue + assert p.rlimit(value) == resource.getrlimit(value) + else: + ret = p.rlimit(value) + assert len(ret) == 2 + assert ret[0] >= -1 + assert ret[1] >= -1 + + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit_set(self): + p = self.spawn_psproc() + p.rlimit(psutil.RLIMIT_NOFILE, (5, 5)) + assert p.rlimit(psutil.RLIMIT_NOFILE) == (5, 5) + # If pid is 0 prlimit() applies to the calling process and + # we don't want that. + if LINUX: + with pytest.raises(ValueError, match="can't use prlimit"): + psutil._psplatform.Process(0).rlimit(0) + with pytest.raises(ValueError): + p.rlimit(psutil.RLIMIT_NOFILE, (5, 5, 5)) + + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit(self): + p = psutil.Process() + testfn = self.get_testfn() + soft, hard = p.rlimit(psutil.RLIMIT_FSIZE) + try: + p.rlimit(psutil.RLIMIT_FSIZE, (1024, hard)) + with open(testfn, "wb") as f: + f.write(b"X" * 1024) + # write() or flush() doesn't always cause the exception + # but close() will. + with pytest.raises(OSError) as exc: + with open(testfn, "wb") as f: + f.write(b"X" * 1025) + assert exc.value.errno == errno.EFBIG + finally: + p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) + assert p.rlimit(psutil.RLIMIT_FSIZE) == (soft, hard) + + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit_infinity(self): + # First set a limit, then re-set it by specifying INFINITY + # and assume we overridden the previous limit. + p = psutil.Process() + soft, hard = p.rlimit(psutil.RLIMIT_FSIZE) + try: + p.rlimit(psutil.RLIMIT_FSIZE, (1024, hard)) + p.rlimit(psutil.RLIMIT_FSIZE, (psutil.RLIM_INFINITY, hard)) + with open(self.get_testfn(), "wb") as f: + f.write(b"X" * 2048) + finally: + p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) + assert p.rlimit(psutil.RLIMIT_FSIZE) == (soft, hard) + + @pytest.mark.skipif(not HAS_RLIMIT, reason="not supported") + def test_rlimit_infinity_value(self): + # RLIMIT_FSIZE should be RLIM_INFINITY, which will be a really + # big number on a platform with large file support. On these + # platforms we need to test that the get/setrlimit functions + # properly convert the number to a C long long and that the + # conversion doesn't raise an error. + p = psutil.Process() + soft, hard = p.rlimit(psutil.RLIMIT_FSIZE) + assert hard == psutil.RLIM_INFINITY + p.rlimit(psutil.RLIMIT_FSIZE, (soft, hard)) + + def test_num_threads(self): + # on certain platforms such as Linux we might test for exact + # thread number, since we always have with 1 thread per process, + # but this does not apply across all platforms (MACOS, Windows) + p = psutil.Process() + if OPENBSD: + try: + step1 = p.num_threads() + except psutil.AccessDenied: + raise pytest.skip("on OpenBSD this requires root access") + else: + step1 = p.num_threads() + + with ThreadTask(): + step2 = p.num_threads() + assert step2 == step1 + 1 + + @pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") + def test_num_handles(self): + # a better test is done later into test/_windows.py + p = psutil.Process() + assert p.num_handles() > 0 + + @pytest.mark.skipif(not HAS_THREADS, reason="not supported") + def test_threads(self): + p = psutil.Process() + if OPENBSD: + try: + step1 = p.threads() + except psutil.AccessDenied: + raise pytest.skip("on OpenBSD this requires root access") + else: + step1 = p.threads() + + with ThreadTask(): + step2 = p.threads() + assert len(step2) == len(step1) + 1 + athread = step2[0] + # test named tuple + assert athread.id == athread[0] + assert athread.user_time == athread[1] + assert athread.system_time == athread[2] + + @retry_on_failure() + @skip_on_access_denied(only_if=MACOS) + @pytest.mark.skipif(not HAS_THREADS, reason="not supported") + def test_threads_2(self): + p = self.spawn_psproc() + if OPENBSD: + try: + p.threads() + except psutil.AccessDenied: + raise pytest.skip("on OpenBSD this requires root access") + assert ( + abs(p.cpu_times().user - sum(x.user_time for x in p.threads())) + < 0.1 + ) + assert ( + abs(p.cpu_times().system - sum(x.system_time for x in p.threads())) + < 0.1 + ) + + @retry_on_failure() + def test_memory_info(self): + p = psutil.Process() + + # step 1 - get a base value to compare our results + rss1, vms1 = p.memory_info()[:2] + percent1 = p.memory_percent() + assert rss1 > 0 + assert vms1 > 0 + + # step 2 - allocate some memory + memarr = [None] * 1500000 + + rss2, vms2 = p.memory_info()[:2] + percent2 = p.memory_percent() + + # step 3 - make sure that the memory usage bumped up + assert rss2 > rss1 + assert vms2 >= vms1 # vms might be equal + assert percent2 > percent1 + del memarr + + if WINDOWS: + mem = p.memory_info() + assert mem.rss == mem.wset + assert mem.vms == mem.pagefile + + mem = p.memory_info() + for name in mem._fields: + assert getattr(mem, name) >= 0 + + def test_memory_full_info(self): + p = psutil.Process() + total = psutil.virtual_memory().total + mem = p.memory_full_info() + for name in mem._fields: + value = getattr(mem, name) + assert value >= 0 + if (name == "vms" and OSX) or LINUX: + continue + assert value <= total + if LINUX or WINDOWS or MACOS: + assert mem.uss >= 0 + if LINUX: + assert mem.pss >= 0 + assert mem.swap >= 0 + + @pytest.mark.skipif(not HAS_MEMORY_MAPS, reason="not supported") + def test_memory_maps(self): + p = psutil.Process() + maps = p.memory_maps() + assert len(maps) == len(set(maps)) + ext_maps = p.memory_maps(grouped=False) + + for nt in maps: + if nt.path.startswith('['): + continue + if BSD and nt.path == "pvclock": + continue + assert os.path.isabs(nt.path), nt.path + + if POSIX: + try: + assert os.path.exists(nt.path) or os.path.islink( + nt.path + ), nt.path + except AssertionError: + if not LINUX: + raise + # https://github.com/giampaolo/psutil/issues/759 + with open_text('/proc/self/smaps') as f: + data = f.read() + if f"{nt.path} (deleted)" not in data: + raise + elif '64' not in os.path.basename(nt.path): + # XXX - On Windows we have this strange behavior with + # 64 bit dlls: they are visible via explorer but cannot + # be accessed via os.stat() (wtf?). + try: + st = os.stat(nt.path) + except FileNotFoundError: + pass + else: + assert stat.S_ISREG(st.st_mode), nt.path + + for nt in ext_maps: + for fname in nt._fields: + value = getattr(nt, fname) + if fname == 'path': + continue + if fname in {'addr', 'perms'}: + assert value, value + else: + assert isinstance(value, int) + assert value >= 0, value + + @pytest.mark.skipif(not HAS_MEMORY_MAPS, reason="not supported") + def test_memory_maps_lists_lib(self): + # Make sure a newly loaded shared lib is listed. + p = psutil.Process() + with copyload_shared_lib() as path: + + def normpath(p): + return os.path.realpath(os.path.normcase(p)) + + libpaths = [normpath(x.path) for x in p.memory_maps()] + assert normpath(path) in libpaths + + def test_memory_percent(self): + p = psutil.Process() + p.memory_percent() + with pytest.raises(ValueError): + p.memory_percent(memtype="?!?") + if LINUX or MACOS or WINDOWS: + p.memory_percent(memtype='uss') + + def test_is_running(self): + p = self.spawn_psproc() + assert p.is_running() + assert p.is_running() + p.kill() + p.wait() + assert not p.is_running() + assert not p.is_running() + + def test_exe(self): + p = self.spawn_psproc() + exe = p.exe() + try: + assert exe == PYTHON_EXE + except AssertionError: + if WINDOWS and len(exe) == len(PYTHON_EXE): + # on Windows we don't care about case sensitivity + normcase = os.path.normcase + assert normcase(exe) == normcase(PYTHON_EXE) + else: + # certain platforms such as BSD are more accurate returning: + # "/usr/local/bin/python3.7" + # ...instead of: + # "/usr/local/bin/python" + # We do not want to consider this difference in accuracy + # an error. + ver = f"{sys.version_info[0]}.{sys.version_info[1]}" + try: + assert exe.replace(ver, '') == PYTHON_EXE.replace(ver, '') + except AssertionError: + # Typically MACOS. Really not sure what to do here. + pass + + out = sh([exe, "-c", "import os; print('hey')"]) + assert out == 'hey' + + def test_cmdline(self): + cmdline = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] + p = self.spawn_psproc(cmdline) + + if NETBSD and p.cmdline() == []: + # https://github.com/giampaolo/psutil/issues/2250 + raise pytest.skip("OPENBSD: returned EBUSY") + + # XXX - most of the times the underlying sysctl() call on Net + # and Open BSD returns a truncated string. + # Also /proc/pid/cmdline behaves the same so it looks + # like this is a kernel bug. + # XXX - AIX truncates long arguments in /proc/pid/cmdline + if NETBSD or OPENBSD or AIX: + assert p.cmdline()[0] == PYTHON_EXE + else: + if MACOS and CI_TESTING: + pyexe = p.cmdline()[0] + if pyexe != PYTHON_EXE: + assert ' '.join(p.cmdline()[1:]) == ' '.join(cmdline[1:]) + return + assert ' '.join(p.cmdline()) == ' '.join(cmdline) + + @pytest.mark.skipif(PYPY, reason="broken on PYPY") + def test_long_cmdline(self): + cmdline = [PYTHON_EXE] + cmdline.extend(["-v"] * 50) + cmdline.extend( + ["-c", "import time; [time.sleep(0.1) for x in range(100)]"] + ) + p = self.spawn_psproc(cmdline) + if OPENBSD: + # XXX: for some reason the test process may turn into a + # zombie (don't know why). + try: + assert p.cmdline() == cmdline + except psutil.ZombieProcess: + raise pytest.skip("OPENBSD: process turned into zombie") + else: + ret = p.cmdline() + if NETBSD and ret == []: + # https://github.com/giampaolo/psutil/issues/2250 + raise pytest.skip("OPENBSD: returned EBUSY") + assert ret == cmdline + + def test_name(self): + p = self.spawn_psproc() + name = p.name().lower() + pyexe = os.path.basename(os.path.realpath(sys.executable)).lower() + assert pyexe.startswith(name), (pyexe, name) + + @pytest.mark.skipif(PYPY, reason="unreliable on PYPY") + def test_long_name(self): + pyexe = create_py_exe(self.get_testfn(suffix=string.digits * 2)) + cmdline = [ + pyexe, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] + p = self.spawn_psproc(cmdline) + if OPENBSD: + # XXX: for some reason the test process may turn into a + # zombie (don't know why). Because the name() is long, all + # UNIX kernels truncate it to 15 chars, so internally psutil + # tries to guess the full name() from the cmdline(). But the + # cmdline() of a zombie on OpenBSD fails (internally), so we + # just compare the first 15 chars. Full explanation: + # https://github.com/giampaolo/psutil/issues/2239 + try: + assert p.name() == os.path.basename(pyexe) + except AssertionError: + if p.status() == psutil.STATUS_ZOMBIE: + assert os.path.basename(pyexe).startswith(p.name()) + else: + raise + else: + assert p.name() == os.path.basename(pyexe) + + # XXX: fails too often + # @pytest.mark.skipif(SUNOS, reason="broken on SUNOS") + # @pytest.mark.skipif(AIX, reason="broken on AIX") + # @pytest.mark.skipif(PYPY, reason="broken on PYPY") + # def test_prog_w_funky_name(self): + # # Test that name(), exe() and cmdline() correctly handle programs + # # with funky chars such as spaces and ")", see: + # # https://github.com/giampaolo/psutil/issues/628 + # pyexe = create_py_exe(self.get_testfn(suffix='foo bar )')) + # cmdline = [ + # pyexe, + # "-c", + # "import time; [time.sleep(0.1) for x in range(100)]", + # ] + # p = self.spawn_psproc(cmdline) + # assert p.cmdline() == cmdline + # assert p.name() == os.path.basename(pyexe) + # assert os.path.normcase(p.exe()) == os.path.normcase(pyexe) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_uids(self): + p = psutil.Process() + real, effective, _saved = p.uids() + # os.getuid() refers to "real" uid + assert real == os.getuid() + # os.geteuid() refers to "effective" uid + assert effective == os.geteuid() + # No such thing as os.getsuid() ("saved" uid), but we have + # os.getresuid() which returns all of them. + if hasattr(os, "getresuid"): + assert os.getresuid() == p.uids() + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_gids(self): + p = psutil.Process() + real, effective, _saved = p.gids() + # os.getuid() refers to "real" uid + assert real == os.getgid() + # os.geteuid() refers to "effective" uid + assert effective == os.getegid() + # No such thing as os.getsgid() ("saved" gid), but we have + # os.getresgid() which returns all of them. + if hasattr(os, "getresuid"): + assert os.getresgid() == p.gids() + + def test_nice(self): + def cleanup(init): + try: + p.nice(init) + except psutil.AccessDenied: + pass + + p = psutil.Process() + with pytest.raises(TypeError): + p.nice("str") + init = p.nice() + self.addCleanup(cleanup, init) + + if WINDOWS: + highest_prio = None + for prio in [ + psutil.IDLE_PRIORITY_CLASS, + psutil.BELOW_NORMAL_PRIORITY_CLASS, + psutil.NORMAL_PRIORITY_CLASS, + psutil.ABOVE_NORMAL_PRIORITY_CLASS, + psutil.HIGH_PRIORITY_CLASS, + psutil.REALTIME_PRIORITY_CLASS, + ]: + with self.subTest(prio=prio): + try: + p.nice(prio) + except psutil.AccessDenied: + pass + else: + new_prio = p.nice() + # The OS may limit our maximum priority, + # even if the function succeeds. For higher + # priorities, we match either the expected + # value or the highest so far. + if prio in { + psutil.ABOVE_NORMAL_PRIORITY_CLASS, + psutil.HIGH_PRIORITY_CLASS, + psutil.REALTIME_PRIORITY_CLASS, + }: + if new_prio == prio or highest_prio is None: + highest_prio = prio + assert new_prio == highest_prio + else: + assert new_prio == prio + else: + try: + if hasattr(os, "getpriority"): + assert ( + os.getpriority(os.PRIO_PROCESS, os.getpid()) + == p.nice() + ) + p.nice(1) + assert p.nice() == 1 + if hasattr(os, "getpriority"): + assert ( + os.getpriority(os.PRIO_PROCESS, os.getpid()) + == p.nice() + ) + # XXX - going back to previous nice value raises + # AccessDenied on MACOS + if not MACOS: + p.nice(0) + assert p.nice() == 0 + except psutil.AccessDenied: + pass + + def test_status(self): + p = psutil.Process() + assert p.status() == psutil.STATUS_RUNNING + + def test_username(self): + p = self.spawn_psproc() + username = p.username() + if WINDOWS: + domain, username = username.split('\\') + getpass_user = getpass.getuser() + if getpass_user.endswith('$'): + # When running as a service account (most likely to be + # NetworkService), these user name calculations don't produce + # the same result, causing the test to fail. + raise pytest.skip('running as service account') + assert username == getpass_user + if 'USERDOMAIN' in os.environ: + assert domain == os.environ['USERDOMAIN'] + else: + assert username == getpass.getuser() + + def test_cwd(self): + p = self.spawn_psproc() + assert p.cwd() == os.getcwd() + + def test_cwd_2(self): + cmd = [ + PYTHON_EXE, + "-c", + ( + "import os, time; os.chdir('..'); [time.sleep(0.1) for x in" + " range(100)]" + ), + ] + p = self.spawn_psproc(cmd) + call_until(lambda: p.cwd() == os.path.dirname(os.getcwd())) + + @pytest.mark.skipif(not HAS_CPU_AFFINITY, reason="not supported") + def test_cpu_affinity(self): + p = psutil.Process() + initial = p.cpu_affinity() + assert initial, initial + self.addCleanup(p.cpu_affinity, initial) + + if hasattr(os, "sched_getaffinity"): + assert initial == list(os.sched_getaffinity(p.pid)) + assert len(initial) == len(set(initial)) + + all_cpus = list(range(len(psutil.cpu_percent(percpu=True)))) + for n in all_cpus: + p.cpu_affinity([n]) + assert p.cpu_affinity() == [n] + if hasattr(os, "sched_getaffinity"): + assert p.cpu_affinity() == list(os.sched_getaffinity(p.pid)) + # also test num_cpu() + if hasattr(p, "num_cpu"): + assert p.cpu_affinity()[0] == p.num_cpu() + + # [] is an alias for "all eligible CPUs"; on Linux this may + # not be equal to all available CPUs, see: + # https://github.com/giampaolo/psutil/issues/956 + p.cpu_affinity([]) + if LINUX: + assert p.cpu_affinity() == p._proc._get_eligible_cpus() + else: + assert p.cpu_affinity() == all_cpus + if hasattr(os, "sched_getaffinity"): + assert p.cpu_affinity() == list(os.sched_getaffinity(p.pid)) + + with pytest.raises(TypeError): + p.cpu_affinity(1) + p.cpu_affinity(initial) + # it should work with all iterables, not only lists + p.cpu_affinity(set(all_cpus)) + p.cpu_affinity(tuple(all_cpus)) + + @pytest.mark.skipif(not HAS_CPU_AFFINITY, reason="not supported") + def test_cpu_affinity_errs(self): + p = self.spawn_psproc() + invalid_cpu = [len(psutil.cpu_times(percpu=True)) + 10] + with pytest.raises(ValueError): + p.cpu_affinity(invalid_cpu) + with pytest.raises(ValueError): + p.cpu_affinity(range(10000, 11000)) + with pytest.raises((TypeError, ValueError)): + p.cpu_affinity([0, "1"]) + with pytest.raises(ValueError): + p.cpu_affinity([0, -1]) + + @pytest.mark.skipif(not HAS_CPU_AFFINITY, reason="not supported") + def test_cpu_affinity_all_combinations(self): + p = psutil.Process() + initial = p.cpu_affinity() + assert initial, initial + self.addCleanup(p.cpu_affinity, initial) + + # All possible CPU set combinations. + if len(initial) > 12: + initial = initial[:12] # ...otherwise it will take forever + combos = [] + for i in range(len(initial) + 1): + combos.extend( + list(subset) + for subset in itertools.combinations(initial, i) + if subset + ) + + for combo in combos: + p.cpu_affinity(combo) + assert sorted(p.cpu_affinity()) == sorted(combo) + + # TODO: #595 + @pytest.mark.skipif(BSD, reason="broken on BSD") + def test_open_files(self): + p = psutil.Process() + testfn = self.get_testfn() + files = p.open_files() + assert testfn not in files + with open(testfn, 'wb') as f: + f.write(b'x' * 1024) + f.flush() + # give the kernel some time to see the new file + call_until(lambda: len(p.open_files()) != len(files)) + files = p.open_files() + filenames = [os.path.normcase(x.path) for x in files] + assert os.path.normcase(testfn) in filenames + if LINUX: + for file in files: + if file.path == testfn: + assert file.position == 1024 + for file in files: + assert os.path.isfile(file.path), file + + # another process + cmdline = ( + f"import time; f = open(r'{testfn}', 'r'); [time.sleep(0.1) for x" + " in range(100)];" + ) + p = self.spawn_psproc([PYTHON_EXE, "-c", cmdline]) + + for x in range(100): + filenames = [os.path.normcase(x.path) for x in p.open_files()] + if testfn in filenames: + break + time.sleep(0.01) + else: + assert os.path.normcase(testfn) in filenames + for file in filenames: + assert os.path.isfile(file), file + + # TODO: #595 + @pytest.mark.skipif(BSD, reason="broken on BSD") + def test_open_files_2(self): + # test fd and path fields + p = psutil.Process() + normcase = os.path.normcase + testfn = self.get_testfn() + with open(testfn, 'w') as fileobj: + for file in p.open_files(): + if ( + normcase(file.path) == normcase(fileobj.name) + or file.fd == fileobj.fileno() + ): + break + else: + raise self.fail(f"no file found; files={p.open_files()!r}") + assert normcase(file.path) == normcase(fileobj.name) + if WINDOWS: + assert file.fd == -1 + else: + assert file.fd == fileobj.fileno() + # test positions + ntuple = p.open_files()[0] + assert ntuple[0] == ntuple.path + assert ntuple[1] == ntuple.fd + # test file is gone + assert fileobj.name not in p.open_files() + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_num_fds(self): + p = psutil.Process() + testfn = self.get_testfn() + start = p.num_fds() + file = open(testfn, 'w') # noqa: SIM115 + self.addCleanup(file.close) + assert p.num_fds() == start + 1 + sock = socket.socket() + self.addCleanup(sock.close) + assert p.num_fds() == start + 2 + file.close() + sock.close() + assert p.num_fds() == start + + @skip_on_not_implemented(only_if=LINUX) + @pytest.mark.skipif( + OPENBSD or NETBSD, reason="not reliable on OPENBSD & NETBSD" + ) + def test_num_ctx_switches(self): + p = psutil.Process() + before = sum(p.num_ctx_switches()) + for _ in range(2): + time.sleep(0.05) # this shall ensure a context switch happens + after = sum(p.num_ctx_switches()) + if after > before: + return + raise self.fail("num ctx switches still the same after 2 iterations") + + def test_ppid(self): + p = psutil.Process() + if hasattr(os, 'getppid'): + assert p.ppid() == os.getppid() + p = self.spawn_psproc() + assert p.ppid() == os.getpid() + + def test_parent(self): + p = self.spawn_psproc() + assert p.parent().pid == os.getpid() + + lowest_pid = psutil.pids()[0] + assert psutil.Process(lowest_pid).parent() is None + + def test_parent_multi(self): + parent = psutil.Process() + child, grandchild = self.spawn_children_pair() + assert grandchild.parent() == child + assert child.parent() == parent + + @retry_on_failure() + def test_parents(self): + parent = psutil.Process() + assert parent.parents() + child, grandchild = self.spawn_children_pair() + assert child.parents()[0] == parent + assert grandchild.parents()[0] == child + assert grandchild.parents()[1] == parent + + def test_children(self): + parent = psutil.Process() + assert not parent.children() + assert not parent.children(recursive=True) + # On Windows we set the flag to 0 in order to cancel out the + # CREATE_NO_WINDOW flag (enabled by default) which creates + # an extra "conhost.exe" child. + child = self.spawn_psproc(creationflags=0) + children1 = parent.children() + children2 = parent.children(recursive=True) + for children in (children1, children2): + assert len(children) == 1 + assert children[0].pid == child.pid + assert children[0].ppid() == parent.pid + + def test_children_recursive(self): + # Test children() against two sub processes, p1 and p2, where + # p1 (our child) spawned p2 (our grandchild). + parent = psutil.Process() + child, grandchild = self.spawn_children_pair() + assert parent.children() == [child] + assert parent.children(recursive=True) == [child, grandchild] + # If the intermediate process is gone there's no way for + # children() to recursively find it. + child.terminate() + child.wait() + assert not parent.children(recursive=True) + + def test_children_duplicates(self): + # find the process which has the highest number of children + table = collections.defaultdict(int) + for p in psutil.process_iter(): + try: + table[p.ppid()] += 1 + except psutil.Error: + pass + # this is the one, now let's make sure there are no duplicates + pid = max(table.items(), key=lambda x: x[1])[0] + if LINUX and pid == 0: + raise pytest.skip("PID 0") + p = psutil.Process(pid) + try: + c = p.children(recursive=True) + except psutil.AccessDenied: # windows + pass + else: + assert len(c) == len(set(c)) + + def test_parents_and_children(self): + parent = psutil.Process() + child, grandchild = self.spawn_children_pair() + # forward + children = parent.children(recursive=True) + assert len(children) == 2 + assert children[0] == child + assert children[1] == grandchild + # backward + parents = grandchild.parents() + assert parents[0] == child + assert parents[1] == parent + + def test_suspend_resume(self): + p = self.spawn_psproc() + p.suspend() + for _ in range(100): + if p.status() == psutil.STATUS_STOPPED: + break + time.sleep(0.01) + p.resume() + assert p.status() != psutil.STATUS_STOPPED + + def test_invalid_pid(self): + with pytest.raises(TypeError): + psutil.Process("1") + with pytest.raises(ValueError): + psutil.Process(-1) + + def test_as_dict(self): + p = psutil.Process() + d = p.as_dict(attrs=['exe', 'name']) + assert sorted(d.keys()) == ['exe', 'name'] + + p = psutil.Process(min(psutil.pids())) + d = p.as_dict(attrs=['net_connections'], ad_value='foo') + if not isinstance(d['net_connections'], list): + assert d['net_connections'] == 'foo' + + # Test ad_value is set on AccessDenied. + with mock.patch( + 'psutil.Process.nice', create=True, side_effect=psutil.AccessDenied + ): + assert p.as_dict(attrs=["nice"], ad_value=1) == {"nice": 1} + + # Test that NoSuchProcess bubbles up. + with mock.patch( + 'psutil.Process.nice', + create=True, + side_effect=psutil.NoSuchProcess(p.pid, "name"), + ): + with pytest.raises(psutil.NoSuchProcess): + p.as_dict(attrs=["nice"]) + + # Test that ZombieProcess is swallowed. + with mock.patch( + 'psutil.Process.nice', + create=True, + side_effect=psutil.ZombieProcess(p.pid, "name"), + ): + assert p.as_dict(attrs=["nice"], ad_value="foo") == {"nice": "foo"} + + # By default APIs raising NotImplementedError are + # supposed to be skipped. + with mock.patch( + 'psutil.Process.nice', create=True, side_effect=NotImplementedError + ): + d = p.as_dict() + assert 'nice' not in list(d.keys()) + # ...unless the user explicitly asked for some attr. + with pytest.raises(NotImplementedError): + p.as_dict(attrs=["nice"]) + + # errors + with pytest.raises(TypeError): + p.as_dict('name') + with pytest.raises(ValueError): + p.as_dict(['foo']) + with pytest.raises(ValueError): + p.as_dict(['foo', 'bar']) + + def test_oneshot(self): + p = psutil.Process() + with mock.patch("psutil._psplatform.Process.cpu_times") as m: + with p.oneshot(): + p.cpu_times() + p.cpu_times() + assert m.call_count == 1 + + with mock.patch("psutil._psplatform.Process.cpu_times") as m: + p.cpu_times() + p.cpu_times() + assert m.call_count == 2 + + def test_oneshot_twice(self): + # Test the case where the ctx manager is __enter__ed twice. + # The second __enter__ is supposed to resut in a NOOP. + p = psutil.Process() + with mock.patch("psutil._psplatform.Process.cpu_times") as m1: + with mock.patch("psutil._psplatform.Process.oneshot_enter") as m2: + with p.oneshot(): + p.cpu_times() + p.cpu_times() + with p.oneshot(): + p.cpu_times() + p.cpu_times() + assert m1.call_count == 1 + assert m2.call_count == 1 + + with mock.patch("psutil._psplatform.Process.cpu_times") as m: + p.cpu_times() + p.cpu_times() + assert m.call_count == 2 + + def test_oneshot_cache(self): + # Make sure oneshot() cache is nonglobal. Instead it's + # supposed to be bound to the Process instance, see: + # https://github.com/giampaolo/psutil/issues/1373 + p1, p2 = self.spawn_children_pair() + p1_ppid = p1.ppid() + p2_ppid = p2.ppid() + assert p1_ppid != p2_ppid + with p1.oneshot(): + assert p1.ppid() == p1_ppid + assert p2.ppid() == p2_ppid + with p2.oneshot(): + assert p1.ppid() == p1_ppid + assert p2.ppid() == p2_ppid + + def test_halfway_terminated_process(self): + # Test that NoSuchProcess exception gets raised in case the + # process dies after we create the Process object. + # Example: + # >>> proc = Process(1234) + # >>> time.sleep(2) # time-consuming task, process dies in meantime + # >>> proc.name() + # Refers to Issue #15 + def assert_raises_nsp(fun, fun_name): + try: + ret = fun() + except psutil.ZombieProcess: # differentiate from NSP + raise + except psutil.NoSuchProcess: + pass + except psutil.AccessDenied: + if OPENBSD and fun_name in {'threads', 'num_threads'}: + return + raise + else: + # NtQuerySystemInformation succeeds even if process is gone. + if WINDOWS and fun_name in {'exe', 'name'}: + return + raise self.fail( + f"{fun!r} didn't raise NSP and returned {ret!r} instead" + ) + + p = self.spawn_psproc() + p.terminate() + p.wait() + if WINDOWS: # XXX + call_until(lambda: p.pid not in psutil.pids()) + self.assertProcessGone(p) + + ns = process_namespace(p) + for fun, name in ns.iter(ns.all): + assert_raises_nsp(fun, name) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_zombie_process(self): + _parent, zombie = self.spawn_zombie() + self.assertProcessZombie(zombie) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_zombie_process_is_running_w_exc(self): + # Emulate a case where internally is_running() raises + # ZombieProcess. + p = psutil.Process() + with mock.patch( + "psutil.Process", side_effect=psutil.ZombieProcess(0) + ) as m: + assert p.is_running() + assert m.called + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_zombie_process_status_w_exc(self): + # Emulate a case where internally status() raises + # ZombieProcess. + p = psutil.Process() + with mock.patch( + "psutil._psplatform.Process.status", + side_effect=psutil.ZombieProcess(0), + ) as m: + assert p.status() == psutil.STATUS_ZOMBIE + assert m.called + + def test_reused_pid(self): + # Emulate a case where PID has been reused by another process. + subp = self.spawn_testproc() + p = psutil.Process(subp.pid) + p._ident = (p.pid, p.create_time() + 100) + + list(psutil.process_iter()) + assert p.pid in psutil._pmap + assert not p.is_running() + + # make sure is_running() removed PID from process_iter() + # internal cache + with mock.patch.object(psutil._common, "PSUTIL_DEBUG", True): + with contextlib.redirect_stderr(io.StringIO()) as f: + list(psutil.process_iter()) + assert ( + f"refreshing Process instance for reused PID {p.pid}" + in f.getvalue() + ) + assert p.pid not in psutil._pmap + + assert p != psutil.Process(subp.pid) + msg = "process no longer exists and its PID has been reused" + ns = process_namespace(p) + for fun, name in ns.iter(ns.setters + ns.killers, clear_cache=False): + with self.subTest(name=name): + with pytest.raises(psutil.NoSuchProcess, match=msg): + fun() + + assert "terminated + PID reused" in str(p) + assert "terminated + PID reused" in repr(p) + + with pytest.raises(psutil.NoSuchProcess, match=msg): + p.ppid() + with pytest.raises(psutil.NoSuchProcess, match=msg): + p.parent() + with pytest.raises(psutil.NoSuchProcess, match=msg): + p.parents() + with pytest.raises(psutil.NoSuchProcess, match=msg): + p.children() + + def test_pid_0(self): + # Process(0) is supposed to work on all platforms except Linux + if 0 not in psutil.pids(): + with pytest.raises(psutil.NoSuchProcess): + psutil.Process(0) + # These 2 are a contradiction, but "ps" says PID 1's parent + # is PID 0. + assert not psutil.pid_exists(0) + assert psutil.Process(1).ppid() == 0 + return + + p = psutil.Process(0) + exc = psutil.AccessDenied if WINDOWS else ValueError + with pytest.raises(exc): + p.wait() + with pytest.raises(exc): + p.terminate() + with pytest.raises(exc): + p.suspend() + with pytest.raises(exc): + p.resume() + with pytest.raises(exc): + p.kill() + with pytest.raises(exc): + p.send_signal(signal.SIGTERM) + + # test all methods + ns = process_namespace(p) + for fun, name in ns.iter(ns.getters + ns.setters): + try: + ret = fun() + except psutil.AccessDenied: + pass + else: + if name in {"uids", "gids"}: + assert ret.real == 0 + elif name == "username": + user = 'NT AUTHORITY\\SYSTEM' if WINDOWS else 'root' + assert p.username() == user + elif name == "name": + assert name, name + + if not OPENBSD: + assert 0 in psutil.pids() + assert psutil.pid_exists(0) + + @pytest.mark.skipif(not HAS_ENVIRON, reason="not supported") + def test_environ(self): + def clean_dict(d): + exclude = ["PLAT", "HOME", "PYTEST_CURRENT_TEST", "PYTEST_VERSION"] + if MACOS: + exclude.extend([ + "__CF_USER_TEXT_ENCODING", + "VERSIONER_PYTHON_PREFER_32_BIT", + "VERSIONER_PYTHON_VERSION", + "VERSIONER_PYTHON_VERSION", + ]) + for name in exclude: + d.pop(name, None) + return { + k.replace("\r", "").replace("\n", ""): v.replace( + "\r", "" + ).replace("\n", "") + for k, v in d.items() + } + + self.maxDiff = None + p = psutil.Process() + d1 = clean_dict(p.environ()) + d2 = clean_dict(os.environ.copy()) + if not OSX and GITHUB_ACTIONS: + assert d1 == d2 + + @pytest.mark.skipif(not HAS_ENVIRON, reason="not supported") + @pytest.mark.skipif(not POSIX, reason="POSIX only") + @pytest.mark.skipif( + MACOS_11PLUS, + reason="macOS 11+ can't get another process environment, issue #2084", + ) + @pytest.mark.skipif( + NETBSD, reason="sometimes fails on `assert is_running()`" + ) + def test_weird_environ(self): + # environment variables can contain values without an equals sign + code = textwrap.dedent(""" + #include + #include + + char * const argv[] = {"cat", 0}; + char * const envp[] = {"A=1", "X", "C=3", 0}; + + int main(void) { + // Close stderr on exec so parent can wait for the + // execve to finish. + if (fcntl(2, F_SETFD, FD_CLOEXEC) != 0) + return 0; + return execve("/bin/cat", argv, envp); + } + """) + cexe = create_c_exe(self.get_testfn(), c_code=code) + sproc = self.spawn_testproc( + [cexe], stdin=subprocess.PIPE, stderr=subprocess.PIPE + ) + p = psutil.Process(sproc.pid) + wait_for_pid(p.pid) + assert p.is_running() + # Wait for process to exec or exit. + assert sproc.stderr.read() == b"" + if MACOS and CI_TESTING: + try: + env = p.environ() + except psutil.AccessDenied: + # XXX: fails sometimes with: + # PermissionError from 'sysctl(KERN_PROCARGS2) -> EIO' + return + else: + env = p.environ() + assert env == {"A": "1", "C": "3"} + sproc.communicate() + assert sproc.returncode == 0 + + +# =================================================================== +# --- psutil.Popen tests +# =================================================================== + + +class TestPopen(PsutilTestCase): + """Tests for psutil.Popen class.""" + + @classmethod + def tearDownClass(cls): + reap_children() + + def test_misc(self): + # XXX this test causes a ResourceWarning because + # psutil.__subproc instance doesn't get properly freed. + # Not sure what to do though. + cmd = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)];", + ] + with psutil.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=PYTHON_EXE_ENV, + ) as proc: + proc.name() + proc.cpu_times() + proc.stdin # noqa: B018 + assert dir(proc) + with pytest.raises(AttributeError): + proc.foo # noqa: B018 + proc.terminate() + if POSIX: + assert proc.wait(5) == -signal.SIGTERM + else: + assert proc.wait(5) == signal.SIGTERM + + def test_ctx_manager(self): + with psutil.Popen( + [PYTHON_EXE, "-V"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + stdin=subprocess.PIPE, + env=PYTHON_EXE_ENV, + ) as proc: + proc.communicate() + assert proc.stdout.closed + assert proc.stderr.closed + assert proc.stdin.closed + assert proc.returncode == 0 + + def test_kill_terminate(self): + # subprocess.Popen()'s terminate(), kill() and send_signal() do + # not raise exception after the process is gone. psutil.Popen + # diverges from that. + cmd = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)];", + ] + with psutil.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=PYTHON_EXE_ENV, + ) as proc: + proc.terminate() + proc.wait() + with pytest.raises(psutil.NoSuchProcess): + proc.terminate() + with pytest.raises(psutil.NoSuchProcess): + proc.kill() + with pytest.raises(psutil.NoSuchProcess): + proc.send_signal(signal.SIGTERM) + if WINDOWS: + with pytest.raises(psutil.NoSuchProcess): + proc.send_signal(signal.CTRL_C_EVENT) + with pytest.raises(psutil.NoSuchProcess): + proc.send_signal(signal.CTRL_BREAK_EVENT) + + def test__getattribute__(self): + cmd = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)];", + ] + with psutil.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=PYTHON_EXE_ENV, + ) as proc: + proc.terminate() + proc.wait() + with pytest.raises(AttributeError): + proc.foo # noqa: B018 diff --git a/lib/python3.12/site-packages/psutil/tests/test_process_all.py b/lib/python3.12/site-packages/psutil/tests/test_process_all.py new file mode 100644 index 0000000000000000000000000000000000000000..aaa3fa01dfd774f163ce0cbc9d358fef2b3b3bd1 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_process_all.py @@ -0,0 +1,535 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Iterate over all process PIDs and for each one of them invoke and +test all psutil.Process() methods. +""" + +import enum +import errno +import multiprocessing +import os +import stat +import time +import traceback + +import psutil +from psutil import AIX +from psutil import BSD +from psutil import FREEBSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import OSX +from psutil import POSIX +from psutil import WINDOWS +from psutil.tests import CI_TESTING +from psutil.tests import PYTEST_PARALLEL +from psutil.tests import VALID_PROC_STATUSES +from psutil.tests import PsutilTestCase +from psutil.tests import check_connection_ntuple +from psutil.tests import create_sockets +from psutil.tests import is_namedtuple +from psutil.tests import is_win_secure_system_proc +from psutil.tests import process_namespace +from psutil.tests import pytest + + +# Cuts the time in half, but (e.g.) on macOS the process pool stays +# alive after join() (multiprocessing bug?), messing up other tests. +USE_PROC_POOL = LINUX and not CI_TESTING and not PYTEST_PARALLEL + + +def proc_info(pid): + tcase = PsutilTestCase() + + def check_exception(exc, proc, name, ppid): + tcase.assertEqual(exc.pid, pid) + if exc.name is not None: + tcase.assertEqual(exc.name, name) + if isinstance(exc, psutil.ZombieProcess): + tcase.assertProcessZombie(proc) + if exc.ppid is not None: + tcase.assertGreaterEqual(exc.ppid, 0) + tcase.assertEqual(exc.ppid, ppid) + elif isinstance(exc, psutil.NoSuchProcess): + tcase.assertProcessGone(proc) + str(exc) + repr(exc) + + def do_wait(): + if pid != 0: + try: + proc.wait(0) + except psutil.Error as exc: + check_exception(exc, proc, name, ppid) + + try: + proc = psutil.Process(pid) + except psutil.NoSuchProcess: + tcase.assertPidGone(pid) + return {} + try: + d = proc.as_dict(['ppid', 'name']) + except psutil.NoSuchProcess: + tcase.assertProcessGone(proc) + else: + name, ppid = d['name'], d['ppid'] + info = {'pid': proc.pid} + ns = process_namespace(proc) + # We don't use oneshot() because in order not to fool + # check_exception() in case of NSP. + for fun, fun_name in ns.iter(ns.getters, clear_cache=False): + try: + info[fun_name] = fun() + except psutil.Error as exc: + check_exception(exc, proc, name, ppid) + continue + do_wait() + return info + + +class TestFetchAllProcesses(PsutilTestCase): + """Test which iterates over all running processes and performs + some sanity checks against Process API's returned values. + Uses a process pool to get info about all processes. + """ + + def setUp(self): + psutil._set_debug(False) + # Using a pool in a CI env may result in deadlock, see: + # https://github.com/giampaolo/psutil/issues/2104 + if USE_PROC_POOL: + self.pool = multiprocessing.Pool() + + def tearDown(self): + psutil._set_debug(True) + if USE_PROC_POOL: + self.pool.terminate() + self.pool.join() + + def iter_proc_info(self): + # Fixes "can't pickle : it's not the + # same object as test_process_all.proc_info". + from psutil.tests.test_process_all import proc_info + + if USE_PROC_POOL: + return self.pool.imap_unordered(proc_info, psutil.pids()) + else: + ls = [proc_info(pid) for pid in psutil.pids()] + return ls + + def test_all(self): + failures = [] + for info in self.iter_proc_info(): + for name, value in info.items(): + meth = getattr(self, name) + try: + meth(value, info) + except Exception: # noqa: BLE001 + s = '\n' + '=' * 70 + '\n' + s += ( + "FAIL: name=test_{}, pid={}, ret={}\ninfo={}\n".format( + name, + info['pid'], + repr(value), + info, + ) + ) + s += '-' * 70 + s += f"\n{traceback.format_exc()}" + s = "\n".join((" " * 4) + i for i in s.splitlines()) + "\n" + failures.append(s) + else: + if value not in (0, 0.0, [], None, '', {}): + assert value, value + if failures: + raise self.fail(''.join(failures)) + + def cmdline(self, ret, info): + assert isinstance(ret, list) + for part in ret: + assert isinstance(part, str) + + def exe(self, ret, info): + assert isinstance(ret, str) + assert ret.strip() == ret + if ret: + if WINDOWS and not ret.endswith('.exe'): + return # May be "Registry", "MemCompression", ... + assert os.path.isabs(ret), ret + # Note: os.stat() may return False even if the file is there + # hence we skip the test, see: + # http://stackoverflow.com/questions/3112546/os-path-exists-lies + if POSIX and os.path.isfile(ret): + if hasattr(os, 'access') and hasattr(os, "X_OK"): + # XXX: may fail on MACOS + try: + assert os.access(ret, os.X_OK) + except AssertionError: + if os.path.exists(ret) and not CI_TESTING: + raise + + def pid(self, ret, info): + assert isinstance(ret, int) + assert ret >= 0 + + def ppid(self, ret, info): + assert isinstance(ret, int) + assert ret >= 0 + proc_info(ret) + + def name(self, ret, info): + assert isinstance(ret, str) + if WINDOWS and not ret and is_win_secure_system_proc(info['pid']): + # https://github.com/giampaolo/psutil/issues/2338 + return + # on AIX, "" processes don't have names + if not AIX: + assert ret, repr(ret) + + def create_time(self, ret, info): + assert isinstance(ret, float) + try: + assert ret >= 0 + except AssertionError: + # XXX + if OPENBSD and info['status'] == psutil.STATUS_ZOMBIE: + pass + else: + raise + # this can't be taken for granted on all platforms + # self.assertGreaterEqual(ret, psutil.boot_time()) + # make sure returned value can be pretty printed + # with strftime + time.strftime("%Y %m %d %H:%M:%S", time.localtime(ret)) + + def uids(self, ret, info): + assert is_namedtuple(ret) + for uid in ret: + assert isinstance(uid, int) + assert uid >= 0 + + def gids(self, ret, info): + assert is_namedtuple(ret) + # note: testing all gids as above seems not to be reliable for + # gid == 30 (nodoby); not sure why. + for gid in ret: + assert isinstance(gid, int) + if not MACOS and not NETBSD: + assert gid >= 0 + + def username(self, ret, info): + assert isinstance(ret, str) + assert ret.strip() == ret + assert ret.strip() + + def status(self, ret, info): + assert isinstance(ret, str) + assert ret, ret + assert ret != '?' # XXX + assert ret in VALID_PROC_STATUSES + + def io_counters(self, ret, info): + assert is_namedtuple(ret) + for field in ret: + assert isinstance(field, int) + if field != -1: + assert field >= 0 + + def ionice(self, ret, info): + if LINUX: + assert isinstance(ret.ioclass, int) + assert isinstance(ret.value, int) + assert ret.ioclass >= 0 + assert ret.value >= 0 + else: # Windows, Cygwin + choices = [ + psutil.IOPRIO_VERYLOW, + psutil.IOPRIO_LOW, + psutil.IOPRIO_NORMAL, + psutil.IOPRIO_HIGH, + ] + assert isinstance(ret, int) + assert ret >= 0 + assert ret in choices + + def num_threads(self, ret, info): + assert isinstance(ret, int) + if WINDOWS and ret == 0 and is_win_secure_system_proc(info['pid']): + # https://github.com/giampaolo/psutil/issues/2338 + return + assert ret >= 1 + + def threads(self, ret, info): + assert isinstance(ret, list) + for t in ret: + assert is_namedtuple(t) + assert t.id >= 0 + assert t.user_time >= 0 + assert t.system_time >= 0 + for field in t: + assert isinstance(field, (int, float)) + + def cpu_times(self, ret, info): + assert is_namedtuple(ret) + for n in ret: + assert isinstance(n, float) + assert n >= 0 + # TODO: check ntuple fields + + def cpu_percent(self, ret, info): + assert isinstance(ret, float) + assert 0.0 <= ret <= 100.0, ret + + def cpu_num(self, ret, info): + assert isinstance(ret, int) + if FREEBSD and ret == -1: + return + assert ret >= 0 + if psutil.cpu_count() == 1: + assert ret == 0 + assert ret in list(range(psutil.cpu_count())) + + def memory_info(self, ret, info): + assert is_namedtuple(ret) + for value in ret: + assert isinstance(value, int) + assert value >= 0 + if WINDOWS: + assert ret.peak_wset >= ret.wset + assert ret.peak_paged_pool >= ret.paged_pool + assert ret.peak_nonpaged_pool >= ret.nonpaged_pool + assert ret.peak_pagefile >= ret.pagefile + + def memory_full_info(self, ret, info): + assert is_namedtuple(ret) + total = psutil.virtual_memory().total + for name in ret._fields: + value = getattr(ret, name) + assert isinstance(value, int) + assert value >= 0 + if LINUX or (OSX and name in {'vms', 'data'}): + # On Linux there are processes (e.g. 'goa-daemon') whose + # VMS is incredibly high for some reason. + continue + assert value <= total, name + + if LINUX: + assert ret.pss >= ret.uss + + def open_files(self, ret, info): + assert isinstance(ret, list) + for f in ret: + assert isinstance(f.fd, int) + assert isinstance(f.path, str) + assert f.path.strip() == f.path + if WINDOWS: + assert f.fd == -1 + elif LINUX: + assert isinstance(f.position, int) + assert isinstance(f.mode, str) + assert isinstance(f.flags, int) + assert f.position >= 0 + assert f.mode in {'r', 'w', 'a', 'r+', 'a+'} + assert f.flags > 0 + elif BSD and not f.path: + # XXX see: https://github.com/giampaolo/psutil/issues/595 + continue + assert os.path.isabs(f.path), f + try: + st = os.stat(f.path) + except FileNotFoundError: + pass + else: + assert stat.S_ISREG(st.st_mode), f + + def num_fds(self, ret, info): + assert isinstance(ret, int) + assert ret >= 0 + + def net_connections(self, ret, info): + with create_sockets(): + assert len(ret) == len(set(ret)) + for conn in ret: + assert is_namedtuple(conn) + check_connection_ntuple(conn) + + def cwd(self, ret, info): + assert isinstance(ret, str) + assert ret.strip() == ret + if ret: + assert os.path.isabs(ret), ret + try: + st = os.stat(ret) + except OSError as err: + if WINDOWS and psutil._psplatform.is_permission_err(err): + pass + # directory has been removed in mean time + elif err.errno != errno.ENOENT: + raise + else: + assert stat.S_ISDIR(st.st_mode) + + def memory_percent(self, ret, info): + assert isinstance(ret, float) + assert 0 <= ret <= 100, ret + + def is_running(self, ret, info): + assert isinstance(ret, bool) + + def cpu_affinity(self, ret, info): + assert isinstance(ret, list) + assert ret != [] + cpus = list(range(psutil.cpu_count())) + for n in ret: + assert isinstance(n, int) + assert n in cpus + + def terminal(self, ret, info): + assert isinstance(ret, (str, type(None))) + if ret is not None: + assert os.path.isabs(ret), ret + assert os.path.exists(ret), ret + + def memory_maps(self, ret, info): + for nt in ret: + assert isinstance(nt.addr, str) + assert isinstance(nt.perms, str) + assert isinstance(nt.path, str) + for fname in nt._fields: + value = getattr(nt, fname) + if fname == 'path': + if value.startswith(("[", "anon_inode:")): # linux + continue + if BSD and value == "pvclock": # seen on FreeBSD + continue + assert os.path.isabs(nt.path), nt.path + # commented as on Linux we might get + # '/foo/bar (deleted)' + # assert os.path.exists(nt.path), nt.path + elif fname == 'addr': + assert value, repr(value) + elif fname == 'perms': + if not WINDOWS: + assert value, repr(value) + else: + assert isinstance(value, int) + assert value >= 0 + + def num_handles(self, ret, info): + assert isinstance(ret, int) + assert ret >= 0 + + def nice(self, ret, info): + assert isinstance(ret, int) + if POSIX: + assert -20 <= ret <= 20, ret + else: + priorities = [ + getattr(psutil, x) + for x in dir(psutil) + if x.endswith('_PRIORITY_CLASS') + ] + assert ret in priorities + assert isinstance(ret, enum.IntEnum) + + def num_ctx_switches(self, ret, info): + assert is_namedtuple(ret) + for value in ret: + assert isinstance(value, int) + assert value >= 0 + + def rlimit(self, ret, info): + assert isinstance(ret, tuple) + assert len(ret) == 2 + assert ret[0] >= -1 + assert ret[1] >= -1 + + def environ(self, ret, info): + assert isinstance(ret, dict) + for k, v in ret.items(): + assert isinstance(k, str) + assert isinstance(v, str) + + +class TestPidsRange(PsutilTestCase): + """Given pid_exists() return value for a range of PIDs which may or + may not exist, make sure that psutil.Process() and psutil.pids() + agree with pid_exists(). This guarantees that the 3 APIs are all + consistent with each other. See: + https://github.com/giampaolo/psutil/issues/2359 + + XXX - Note about Windows: it turns out there are some "hidden" PIDs + which are not returned by psutil.pids() and are also not revealed + by taskmgr.exe and ProcessHacker, still they can be instantiated by + psutil.Process() and queried. One of such PIDs is "conhost.exe". + Running as_dict() for it reveals that some Process() APIs + erroneously raise NoSuchProcess, so we know we have problem there. + Let's ignore this for now, since it's quite a corner case (who even + imagined hidden PIDs existed on Windows?). + """ + + def setUp(self): + psutil._set_debug(False) + + def tearDown(self): + psutil._set_debug(True) + + def test_it(self): + def is_linux_tid(pid): + try: + f = open(f"/proc/{pid}/status", "rb") # noqa: SIM115 + except FileNotFoundError: + return False + else: + with f: + for line in f: + if line.startswith(b"Tgid:"): + tgid = int(line.split()[1]) + # If tgid and pid are different then we're + # dealing with a process TID. + return tgid != pid + raise ValueError("'Tgid' line not found") + + def check(pid): + # In case of failure retry up to 3 times in order to avoid + # race conditions, especially when running in a CI + # environment where PIDs may appear and disappear at any + # time. + x = 3 + while True: + exists = psutil.pid_exists(pid) + try: + if exists: + psutil.Process(pid) + if not WINDOWS: # see docstring + assert pid in psutil.pids() + else: + # On OpenBSD thread IDs can be instantiated, + # and oneshot() succeeds, but other APIs fail + # with EINVAL. + if not OPENBSD: + with pytest.raises(psutil.NoSuchProcess): + psutil.Process(pid) + if not WINDOWS: # see docstring + assert pid not in psutil.pids() + except (psutil.Error, AssertionError): + x -= 1 + if x == 0: + raise + else: + return + + for pid in range(1, 3000): + if LINUX and is_linux_tid(pid): + # On Linux a TID (thread ID) can be passed to the + # Process class and is querable like a PID (process + # ID). Skip it. + continue + with self.subTest(pid=pid): + check(pid) diff --git a/lib/python3.12/site-packages/psutil/tests/test_scripts.py b/lib/python3.12/site-packages/psutil/tests/test_scripts.py new file mode 100644 index 0000000000000000000000000000000000000000..de0ad2af747cd1daec63f4cc5e1320e855cfcfad --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_scripts.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Test various scripts.""" + +import ast +import os +import shutil +import stat +import subprocess + +import pytest + +from psutil import POSIX +from psutil import WINDOWS +from psutil.tests import CI_TESTING +from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV +from psutil.tests import ROOT_DIR +from psutil.tests import SCRIPTS_DIR +from psutil.tests import PsutilTestCase +from psutil.tests import import_module_by_path +from psutil.tests import psutil +from psutil.tests import sh + + +INTERNAL_SCRIPTS_DIR = os.path.join(SCRIPTS_DIR, "internal") +SETUP_PY = os.path.join(ROOT_DIR, 'setup.py') + + +# =================================================================== +# --- Tests scripts in scripts/ directory +# =================================================================== + + +@pytest.mark.skipif( + CI_TESTING and not os.path.exists(SCRIPTS_DIR), + reason="can't find scripts/ directory", +) +class TestExampleScripts(PsutilTestCase): + @staticmethod + def assert_stdout(exe, *args, **kwargs): + kwargs.setdefault("env", PYTHON_EXE_ENV) + exe = os.path.join(SCRIPTS_DIR, exe) + cmd = [PYTHON_EXE, exe] + for arg in args: + cmd.append(arg) + try: + out = sh(cmd, **kwargs).strip() + except RuntimeError as err: + if 'AccessDenied' in str(err): + return str(err) + else: + raise + assert out, out + return out + + @staticmethod + def assert_syntax(exe): + exe = os.path.join(SCRIPTS_DIR, exe) + with open(exe, encoding="utf8") as f: + src = f.read() + ast.parse(src) + + def test_coverage(self): + # make sure all example scripts have a test method defined + meths = dir(self) + for name in os.listdir(SCRIPTS_DIR): + if name.endswith('.py'): + if 'test_' + os.path.splitext(name)[0] not in meths: + # self.assert_stdout(name) + raise self.fail( + "no test defined for" + f" {os.path.join(SCRIPTS_DIR, name)!r} script" + ) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_executable(self): + for root, dirs, files in os.walk(SCRIPTS_DIR): + for file in files: + if file.endswith('.py'): + path = os.path.join(root, file) + if not stat.S_IXUSR & os.stat(path)[stat.ST_MODE]: + raise self.fail(f"{path!r} is not executable") + + def test_disk_usage(self): + self.assert_stdout('disk_usage.py') + + def test_free(self): + self.assert_stdout('free.py') + + def test_meminfo(self): + self.assert_stdout('meminfo.py') + + def test_procinfo(self): + self.assert_stdout('procinfo.py', str(os.getpid())) + + @pytest.mark.skipif(CI_TESTING and not psutil.users(), reason="no users") + def test_who(self): + self.assert_stdout('who.py') + + def test_ps(self): + self.assert_stdout('ps.py') + + def test_pstree(self): + self.assert_stdout('pstree.py') + + def test_netstat(self): + self.assert_stdout('netstat.py') + + def test_ifconfig(self): + self.assert_stdout('ifconfig.py') + + @pytest.mark.skipif(not HAS_MEMORY_MAPS, reason="not supported") + def test_pmap(self): + self.assert_stdout('pmap.py', str(os.getpid())) + + def test_procsmem(self): + if 'uss' not in psutil.Process().memory_full_info()._fields: + raise pytest.skip("not supported") + self.assert_stdout('procsmem.py') + + def test_killall(self): + self.assert_syntax('killall.py') + + def test_nettop(self): + self.assert_syntax('nettop.py') + + def test_top(self): + self.assert_syntax('top.py') + + def test_iotop(self): + self.assert_syntax('iotop.py') + + def test_pidof(self): + output = self.assert_stdout('pidof.py', psutil.Process().name()) + assert str(os.getpid()) in output + + @pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") + def test_winservices(self): + self.assert_stdout('winservices.py') + + def test_cpu_distribution(self): + self.assert_syntax('cpu_distribution.py') + + @pytest.mark.skipif(not HAS_SENSORS_TEMPERATURES, reason="not supported") + def test_temperatures(self): + if not psutil.sensors_temperatures(): + raise pytest.skip("no temperatures") + self.assert_stdout('temperatures.py') + + @pytest.mark.skipif(not HAS_SENSORS_FANS, reason="not supported") + def test_fans(self): + if not psutil.sensors_fans(): + raise pytest.skip("no fans") + self.assert_stdout('fans.py') + + @pytest.mark.skipif(not HAS_SENSORS_BATTERY, reason="not supported") + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_battery(self): + self.assert_stdout('battery.py') + + @pytest.mark.skipif(not HAS_SENSORS_BATTERY, reason="not supported") + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_sensors(self): + self.assert_stdout('sensors.py') + + +# =================================================================== +# --- Tests scripts in scripts/internal/ directory +# =================================================================== + + +@pytest.mark.skipif( + CI_TESTING and not os.path.exists(INTERNAL_SCRIPTS_DIR), + reason="can't find scripts/internal/ directory", +) +class TestInternalScripts(PsutilTestCase): + @staticmethod + def ls(): + for name in os.listdir(INTERNAL_SCRIPTS_DIR): + if name.endswith(".py"): + yield os.path.join(INTERNAL_SCRIPTS_DIR, name) + + def test_syntax_all(self): + for path in self.ls(): + with open(path, encoding="utf8") as f: + data = f.read() + ast.parse(data) + + @pytest.mark.skipif(CI_TESTING, reason="not on CI") + def test_import_all(self): + for path in self.ls(): + try: + import_module_by_path(path) + except SystemExit: + pass + + +# =================================================================== +# --- Tests for setup.py script +# =================================================================== + + +@pytest.mark.skipif( + CI_TESTING and not os.path.exists(SETUP_PY), reason="can't find setup.py" +) +class TestSetupScript(PsutilTestCase): + def test_invocation(self): + module = import_module_by_path(SETUP_PY) + with pytest.raises(SystemExit): + module.setup() + assert module.get_version() == psutil.__version__ + + @pytest.mark.skipif( + not shutil.which("python2.7"), reason="python2.7 not installed" + ) + def test_python2(self): + # There's a duplicate of this test in scripts/internal + # directory, which is only executed by CI. We replicate it here + # to run it when developing locally. + p = subprocess.Popen( + [shutil.which("python2.7"), SETUP_PY], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True, + ) + stdout, stderr = p.communicate() + assert p.wait() == 1 + assert not stdout + assert "psutil no longer supports Python 2.7" in stderr + assert "Latest version supporting Python 2.7 is" in stderr diff --git a/lib/python3.12/site-packages/psutil/tests/test_sunos.py b/lib/python3.12/site-packages/psutil/tests/test_sunos.py new file mode 100644 index 0000000000000000000000000000000000000000..b5d9d353b94e60f537b967b7a9fbcfc13cf5bf65 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_sunos.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Sun OS specific tests.""" + +import os + +import psutil +from psutil import SUNOS +from psutil.tests import PsutilTestCase +from psutil.tests import pytest +from psutil.tests import sh + + +@pytest.mark.skipif(not SUNOS, reason="SUNOS only") +class SunOSSpecificTestCase(PsutilTestCase): + def test_swap_memory(self): + out = sh(f"env PATH=/usr/sbin:/sbin:{os.environ['PATH']} swap -l") + lines = out.strip().split('\n')[1:] + if not lines: + raise ValueError('no swap device(s) configured') + total = free = 0 + for line in lines: + fields = line.split() + total = int(fields[3]) * 512 + free = int(fields[4]) * 512 + used = total - free + + psutil_swap = psutil.swap_memory() + assert psutil_swap.total == total + assert psutil_swap.used == used + assert psutil_swap.free == free + + def test_cpu_count(self): + out = sh("/usr/sbin/psrinfo") + assert psutil.cpu_count() == len(out.split('\n')) diff --git a/lib/python3.12/site-packages/psutil/tests/test_system.py b/lib/python3.12/site-packages/psutil/tests/test_system.py new file mode 100644 index 0000000000000000000000000000000000000000..b961e1f89178932ce99ea3a6cc1acfa745bfa280 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_system.py @@ -0,0 +1,979 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for system APIS.""" + +import datetime +import enum +import errno +import os +import platform +import pprint +import shutil +import signal +import socket +import sys +import time +from unittest import mock + +import psutil +from psutil import AIX +from psutil import BSD +from psutil import FREEBSD +from psutil import LINUX +from psutil import MACOS +from psutil import NETBSD +from psutil import OPENBSD +from psutil import POSIX +from psutil import SUNOS +from psutil import WINDOWS +from psutil._common import broadcast_addr +from psutil.tests import AARCH64 +from psutil.tests import ASCII_FS +from psutil.tests import CI_TESTING +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import GLOBAL_TIMEOUT +from psutil.tests import HAS_BATTERY +from psutil.tests import HAS_CPU_FREQ +from psutil.tests import HAS_GETLOADAVG +from psutil.tests import HAS_NET_IO_COUNTERS +from psutil.tests import HAS_SENSORS_BATTERY +from psutil.tests import HAS_SENSORS_FANS +from psutil.tests import HAS_SENSORS_TEMPERATURES +from psutil.tests import IS_64BIT +from psutil.tests import MACOS_12PLUS +from psutil.tests import PYPY +from psutil.tests import UNICODE_SUFFIX +from psutil.tests import PsutilTestCase +from psutil.tests import check_net_address +from psutil.tests import pytest +from psutil.tests import retry_on_failure + + +# =================================================================== +# --- System-related API tests +# =================================================================== + + +class TestProcessIter(PsutilTestCase): + def test_pid_presence(self): + assert os.getpid() in [x.pid for x in psutil.process_iter()] + sproc = self.spawn_testproc() + assert sproc.pid in [x.pid for x in psutil.process_iter()] + p = psutil.Process(sproc.pid) + p.kill() + p.wait() + assert sproc.pid not in [x.pid for x in psutil.process_iter()] + + def test_no_duplicates(self): + ls = list(psutil.process_iter()) + assert sorted(ls, key=lambda x: x.pid) == sorted( + set(ls), key=lambda x: x.pid + ) + + def test_emulate_nsp(self): + list(psutil.process_iter()) # populate cache + for x in range(2): + with mock.patch( + 'psutil.Process.as_dict', + side_effect=psutil.NoSuchProcess(os.getpid()), + ): + assert not list(psutil.process_iter(attrs=["cpu_times"])) + psutil.process_iter.cache_clear() # repeat test without cache + + def test_emulate_access_denied(self): + list(psutil.process_iter()) # populate cache + for x in range(2): + with mock.patch( + 'psutil.Process.as_dict', + side_effect=psutil.AccessDenied(os.getpid()), + ): + with pytest.raises(psutil.AccessDenied): + list(psutil.process_iter(attrs=["cpu_times"])) + psutil.process_iter.cache_clear() # repeat test without cache + + def test_attrs(self): + for p in psutil.process_iter(attrs=['pid']): + assert list(p.info.keys()) == ['pid'] + # yield again + for p in psutil.process_iter(attrs=['pid']): + assert list(p.info.keys()) == ['pid'] + with pytest.raises(ValueError): + list(psutil.process_iter(attrs=['foo'])) + with mock.patch( + "psutil._psplatform.Process.cpu_times", + side_effect=psutil.AccessDenied(0, ""), + ) as m: + for p in psutil.process_iter(attrs=["pid", "cpu_times"]): + assert p.info['cpu_times'] is None + assert p.info['pid'] >= 0 + assert m.called + with mock.patch( + "psutil._psplatform.Process.cpu_times", + side_effect=psutil.AccessDenied(0, ""), + ) as m: + flag = object() + for p in psutil.process_iter( + attrs=["pid", "cpu_times"], ad_value=flag + ): + assert p.info['cpu_times'] is flag + assert p.info['pid'] >= 0 + assert m.called + + def test_cache_clear(self): + list(psutil.process_iter()) # populate cache + assert psutil._pmap + psutil.process_iter.cache_clear() + assert not psutil._pmap + + +class TestProcessAPIs(PsutilTestCase): + @pytest.mark.skipif( + PYPY and WINDOWS, + reason="spawn_testproc() unreliable on PYPY + WINDOWS", + ) + def test_wait_procs(self): + def callback(p): + pids.append(p.pid) + + pids = [] + sproc1 = self.spawn_testproc() + sproc2 = self.spawn_testproc() + sproc3 = self.spawn_testproc() + procs = [psutil.Process(x.pid) for x in (sproc1, sproc2, sproc3)] + with pytest.raises(ValueError): + psutil.wait_procs(procs, timeout=-1) + with pytest.raises(TypeError): + psutil.wait_procs(procs, callback=1) + t = time.time() + gone, alive = psutil.wait_procs(procs, timeout=0.01, callback=callback) + + assert time.time() - t < 0.5 + assert not gone + assert len(alive) == 3 + assert not pids + for p in alive: + assert not hasattr(p, 'returncode') + + @retry_on_failure(30) + def test_1(procs, callback): + gone, alive = psutil.wait_procs( + procs, timeout=0.03, callback=callback + ) + assert len(gone) == 1 + assert len(alive) == 2 + return gone, alive + + sproc3.terminate() + gone, alive = test_1(procs, callback) + assert sproc3.pid in [x.pid for x in gone] + if POSIX: + assert gone.pop().returncode == -signal.SIGTERM + else: + assert gone.pop().returncode == 1 + assert pids == [sproc3.pid] + for p in alive: + assert not hasattr(p, 'returncode') + + @retry_on_failure(30) + def test_2(procs, callback): + gone, alive = psutil.wait_procs( + procs, timeout=0.03, callback=callback + ) + assert len(gone) == 3 + assert len(alive) == 0 + return gone, alive + + sproc1.terminate() + sproc2.terminate() + gone, alive = test_2(procs, callback) + assert set(pids) == {sproc1.pid, sproc2.pid, sproc3.pid} + for p in gone: + assert hasattr(p, 'returncode') + + @pytest.mark.skipif( + PYPY and WINDOWS, + reason="spawn_testproc() unreliable on PYPY + WINDOWS", + ) + def test_wait_procs_no_timeout(self): + sproc1 = self.spawn_testproc() + sproc2 = self.spawn_testproc() + sproc3 = self.spawn_testproc() + procs = [psutil.Process(x.pid) for x in (sproc1, sproc2, sproc3)] + for p in procs: + p.terminate() + psutil.wait_procs(procs) + + def test_pid_exists(self): + sproc = self.spawn_testproc() + assert psutil.pid_exists(sproc.pid) + p = psutil.Process(sproc.pid) + p.kill() + p.wait() + assert not psutil.pid_exists(sproc.pid) + assert not psutil.pid_exists(-1) + assert psutil.pid_exists(0) == (0 in psutil.pids()) + + def test_pid_exists_2(self): + pids = psutil.pids() + for pid in pids: + try: + assert psutil.pid_exists(pid) + except AssertionError: + # in case the process disappeared in meantime fail only + # if it is no longer in psutil.pids() + time.sleep(0.1) + assert pid not in psutil.pids() + pids = range(max(pids) + 15000, max(pids) + 16000) + for pid in pids: + assert not psutil.pid_exists(pid) + + +class TestMiscAPIs(PsutilTestCase): + def test_boot_time(self): + bt = psutil.boot_time() + assert isinstance(bt, float) + assert bt > 0 + assert bt < time.time() + + @pytest.mark.skipif( + CI_TESTING and not psutil.users(), reason="unreliable on CI" + ) + def test_users(self): + users = psutil.users() + assert users + for user in users: + with self.subTest(user=user): + assert user.name + assert isinstance(user.name, str) + assert isinstance(user.terminal, (str, type(None))) + if user.host is not None: + assert isinstance(user.host, (str, type(None))) + user.terminal # noqa: B018 + user.host # noqa: B018 + assert user.started > 0.0 + datetime.datetime.fromtimestamp(user.started) + if WINDOWS or OPENBSD: + assert user.pid is None + else: + psutil.Process(user.pid) + + def test_os_constants(self): + names = [ + "POSIX", + "WINDOWS", + "LINUX", + "MACOS", + "FREEBSD", + "OPENBSD", + "NETBSD", + "BSD", + "SUNOS", + ] + for name in names: + assert isinstance(getattr(psutil, name), bool), name + + if os.name == 'posix': + assert psutil.POSIX + assert not psutil.WINDOWS + names.remove("POSIX") + if "linux" in sys.platform.lower(): + assert psutil.LINUX + names.remove("LINUX") + elif "bsd" in sys.platform.lower(): + assert psutil.BSD + assert [psutil.FREEBSD, psutil.OPENBSD, psutil.NETBSD].count( + True + ) == 1 + names.remove("BSD") + names.remove("FREEBSD") + names.remove("OPENBSD") + names.remove("NETBSD") + elif ( + "sunos" in sys.platform.lower() + or "solaris" in sys.platform.lower() + ): + assert psutil.SUNOS + names.remove("SUNOS") + elif "darwin" in sys.platform.lower(): + assert psutil.MACOS + names.remove("MACOS") + else: + assert psutil.WINDOWS + assert not psutil.POSIX + names.remove("WINDOWS") + + # assert all other constants are set to False + for name in names: + assert not getattr(psutil, name), name + + +class TestMemoryAPIs(PsutilTestCase): + def test_virtual_memory(self): + mem = psutil.virtual_memory() + assert mem.total > 0, mem + assert mem.available > 0, mem + assert 0 <= mem.percent <= 100, mem + assert mem.used > 0, mem + assert mem.free >= 0, mem + for name in mem._fields: + value = getattr(mem, name) + if name != 'percent': + assert isinstance(value, int) + if name != 'total': + if not value >= 0: + raise self.fail(f"{name!r} < 0 ({value})") + if value > mem.total: + raise self.fail( + f"{name!r} > total (total={mem.total}, {name}={value})" + ) + + def test_swap_memory(self): + mem = psutil.swap_memory() + assert mem._fields == ( + 'total', + 'used', + 'free', + 'percent', + 'sin', + 'sout', + ) + + assert mem.total >= 0, mem + assert mem.used >= 0, mem + if mem.total > 0: + # likely a system with no swap partition + assert mem.free > 0, mem + else: + assert mem.free == 0, mem + assert 0 <= mem.percent <= 100, mem + assert mem.sin >= 0, mem + assert mem.sout >= 0, mem + + +class TestCpuAPIs(PsutilTestCase): + def test_cpu_count_logical(self): + logical = psutil.cpu_count() + assert logical is not None + assert logical == len(psutil.cpu_times(percpu=True)) + assert logical >= 1 + + if os.path.exists("/proc/cpuinfo"): + with open("/proc/cpuinfo") as fd: + cpuinfo_data = fd.read() + if "physical id" not in cpuinfo_data: + raise pytest.skip("cpuinfo doesn't include physical id") + + def test_cpu_count_cores(self): + logical = psutil.cpu_count() + cores = psutil.cpu_count(logical=False) + if cores is None: + raise pytest.skip("cpu_count_cores() is None") + if WINDOWS and sys.getwindowsversion()[:2] <= (6, 1): # <= Vista + assert cores is None + else: + assert cores >= 1 + assert logical >= cores + + def test_cpu_count_none(self): + # https://github.com/giampaolo/psutil/issues/1085 + for val in (-1, 0, None): + with mock.patch( + 'psutil._psplatform.cpu_count_logical', return_value=val + ) as m: + assert psutil.cpu_count() is None + assert m.called + with mock.patch( + 'psutil._psplatform.cpu_count_cores', return_value=val + ) as m: + assert psutil.cpu_count(logical=False) is None + assert m.called + + def test_cpu_times(self): + # Check type, value >= 0, str(). + total = 0 + times = psutil.cpu_times() + sum(times) + for cp_time in times: + assert isinstance(cp_time, float) + assert cp_time >= 0.0 + total += cp_time + assert round(abs(total - sum(times)), 6) == 0 + str(times) + # CPU times are always supposed to increase over time + # or at least remain the same and that's because time + # cannot go backwards. + # Surprisingly sometimes this might not be the case (at + # least on Windows and Linux), see: + # https://github.com/giampaolo/psutil/issues/392 + # https://github.com/giampaolo/psutil/issues/645 + # if not WINDOWS: + # last = psutil.cpu_times() + # for x in range(100): + # new = psutil.cpu_times() + # for field in new._fields: + # new_t = getattr(new, field) + # last_t = getattr(last, field) + # self.assertGreaterEqual( + # new_t, last_t, + # msg="{} {}".format(new_t, last_t)) + # last = new + + def test_cpu_times_time_increases(self): + # Make sure time increases between calls. + t1 = sum(psutil.cpu_times()) + stop_at = time.time() + GLOBAL_TIMEOUT + while time.time() < stop_at: + t2 = sum(psutil.cpu_times()) + if t2 > t1: + return + raise self.fail("time remained the same") + + def test_per_cpu_times(self): + # Check type, value >= 0, str(). + for times in psutil.cpu_times(percpu=True): + total = 0 + sum(times) + for cp_time in times: + assert isinstance(cp_time, float) + assert cp_time >= 0.0 + total += cp_time + assert round(abs(total - sum(times)), 6) == 0 + str(times) + assert len(psutil.cpu_times(percpu=True)[0]) == len( + psutil.cpu_times(percpu=False) + ) + + # Note: in theory CPU times are always supposed to increase over + # time or remain the same but never go backwards. In practice + # sometimes this is not the case. + # This issue seemd to be afflict Windows: + # https://github.com/giampaolo/psutil/issues/392 + # ...but it turns out also Linux (rarely) behaves the same. + # last = psutil.cpu_times(percpu=True) + # for x in range(100): + # new = psutil.cpu_times(percpu=True) + # for index in range(len(new)): + # newcpu = new[index] + # lastcpu = last[index] + # for field in newcpu._fields: + # new_t = getattr(newcpu, field) + # last_t = getattr(lastcpu, field) + # self.assertGreaterEqual( + # new_t, last_t, msg="{} {}".format(lastcpu, newcpu)) + # last = new + + def test_per_cpu_times_2(self): + # Simulate some work load then make sure time have increased + # between calls. + tot1 = psutil.cpu_times(percpu=True) + giveup_at = time.time() + GLOBAL_TIMEOUT + while True: + if time.time() >= giveup_at: + return self.fail("timeout") + tot2 = psutil.cpu_times(percpu=True) + for t1, t2 in zip(tot1, tot2): + t1, t2 = psutil._cpu_busy_time(t1), psutil._cpu_busy_time(t2) + difference = t2 - t1 + if difference >= 0.05: + return None + + @pytest.mark.skipif( + CI_TESTING and OPENBSD, reason="unreliable on OPENBSD + CI" + ) + @retry_on_failure(30) + def test_cpu_times_comparison(self): + # Make sure the sum of all per cpu times is almost equal to + # base "one cpu" times. On OpenBSD the sum of per-CPUs is + # higher for some reason. + base = psutil.cpu_times() + per_cpu = psutil.cpu_times(percpu=True) + summed_values = base._make([sum(num) for num in zip(*per_cpu)]) + for field in base._fields: + with self.subTest(field=field, base=base, per_cpu=per_cpu): + assert ( + abs(getattr(base, field) - getattr(summed_values, field)) + < 2 + ) + + def _test_cpu_percent(self, percent, last_ret, new_ret): + try: + assert isinstance(percent, float) + assert percent >= 0.0 + assert percent <= 100.0 * psutil.cpu_count() + except AssertionError as err: + raise AssertionError( + "\n{}\nlast={}\nnew={}".format( + err, pprint.pformat(last_ret), pprint.pformat(new_ret) + ) + ) + + def test_cpu_percent(self): + last = psutil.cpu_percent(interval=0.001) + for _ in range(100): + new = psutil.cpu_percent(interval=None) + self._test_cpu_percent(new, last, new) + last = new + with pytest.raises(ValueError): + psutil.cpu_percent(interval=-1) + + def test_per_cpu_percent(self): + last = psutil.cpu_percent(interval=0.001, percpu=True) + assert len(last) == psutil.cpu_count() + for _ in range(100): + new = psutil.cpu_percent(interval=None, percpu=True) + for percent in new: + self._test_cpu_percent(percent, last, new) + last = new + with pytest.raises(ValueError): + psutil.cpu_percent(interval=-1, percpu=True) + + def test_cpu_times_percent(self): + last = psutil.cpu_times_percent(interval=0.001) + for _ in range(100): + new = psutil.cpu_times_percent(interval=None) + for percent in new: + self._test_cpu_percent(percent, last, new) + self._test_cpu_percent(sum(new), last, new) + last = new + with pytest.raises(ValueError): + psutil.cpu_times_percent(interval=-1) + + def test_per_cpu_times_percent(self): + last = psutil.cpu_times_percent(interval=0.001, percpu=True) + assert len(last) == psutil.cpu_count() + for _ in range(100): + new = psutil.cpu_times_percent(interval=None, percpu=True) + for cpu in new: + for percent in cpu: + self._test_cpu_percent(percent, last, new) + self._test_cpu_percent(sum(cpu), last, new) + last = new + + def test_per_cpu_times_percent_negative(self): + # see: https://github.com/giampaolo/psutil/issues/645 + psutil.cpu_times_percent(percpu=True) + zero_times = [ + x._make([0 for x in range(len(x._fields))]) + for x in psutil.cpu_times(percpu=True) + ] + with mock.patch('psutil.cpu_times', return_value=zero_times): + for cpu in psutil.cpu_times_percent(percpu=True): + for percent in cpu: + self._test_cpu_percent(percent, None, None) + + def test_cpu_stats(self): + # Tested more extensively in per-platform test modules. + infos = psutil.cpu_stats() + assert infos._fields == ( + 'ctx_switches', + 'interrupts', + 'soft_interrupts', + 'syscalls', + ) + for name in infos._fields: + value = getattr(infos, name) + assert value >= 0 + # on AIX, ctx_switches is always 0 + if not AIX and name in {'ctx_switches', 'interrupts'}: + assert value > 0 + + # TODO: remove this once 1892 is fixed + @pytest.mark.skipif( + MACOS and platform.machine() == 'arm64', reason="skipped due to #1892" + ) + @pytest.mark.skipif(not HAS_CPU_FREQ, reason="not supported") + def test_cpu_freq(self): + def check_ls(ls): + for nt in ls: + assert nt._fields == ('current', 'min', 'max') + if nt.max != 0.0: + assert nt.current <= nt.max + for name in nt._fields: + value = getattr(nt, name) + assert isinstance(value, (int, float)) + assert value >= 0 + + ls = psutil.cpu_freq(percpu=True) + if (FREEBSD or AARCH64) and not ls: + raise pytest.skip( + "returns empty list on FreeBSD and Linux aarch64" + ) + + assert ls, ls + check_ls([psutil.cpu_freq(percpu=False)]) + + if LINUX: + assert len(ls) == psutil.cpu_count() + + @pytest.mark.skipif(not HAS_GETLOADAVG, reason="not supported") + def test_getloadavg(self): + loadavg = psutil.getloadavg() + assert len(loadavg) == 3 + for load in loadavg: + assert isinstance(load, float) + assert load >= 0.0 + + +class TestDiskAPIs(PsutilTestCase): + @pytest.mark.skipif( + PYPY and not IS_64BIT, reason="unreliable on PYPY32 + 32BIT" + ) + def test_disk_usage(self): + usage = psutil.disk_usage(os.getcwd()) + assert usage._fields == ('total', 'used', 'free', 'percent') + + assert usage.total > 0, usage + assert usage.used > 0, usage + assert usage.free > 0, usage + assert usage.total > usage.used, usage + assert usage.total > usage.free, usage + assert 0 <= usage.percent <= 100, usage.percent + if hasattr(shutil, 'disk_usage'): + # py >= 3.3, see: http://bugs.python.org/issue12442 + shutil_usage = shutil.disk_usage(os.getcwd()) + tolerance = 5 * 1024 * 1024 # 5MB + assert usage.total == shutil_usage.total + assert abs(usage.free - shutil_usage.free) < tolerance + if not MACOS_12PLUS: + # see https://github.com/giampaolo/psutil/issues/2147 + assert abs(usage.used - shutil_usage.used) < tolerance + + # if path does not exist OSError ENOENT is expected across + # all platforms + fname = self.get_testfn() + with pytest.raises(FileNotFoundError): + psutil.disk_usage(fname) + + @pytest.mark.skipif(not ASCII_FS, reason="not an ASCII fs") + def test_disk_usage_unicode(self): + # See: https://github.com/giampaolo/psutil/issues/416 + with pytest.raises(UnicodeEncodeError): + psutil.disk_usage(UNICODE_SUFFIX) + + def test_disk_usage_bytes(self): + psutil.disk_usage(b'.') + + def test_disk_partitions(self): + def check_ntuple(nt): + assert isinstance(nt.device, str) + assert isinstance(nt.mountpoint, str) + assert isinstance(nt.fstype, str) + assert isinstance(nt.opts, str) + + # all = False + ls = psutil.disk_partitions(all=False) + assert ls + for disk in ls: + check_ntuple(disk) + if WINDOWS and 'cdrom' in disk.opts: + continue + if not POSIX: + assert os.path.exists(disk.device), disk + else: + # we cannot make any assumption about this, see: + # http://goo.gl/p9c43 + disk.device # noqa: B018 + # on modern systems mount points can also be files + assert os.path.exists(disk.mountpoint), disk + assert disk.fstype, disk + + # all = True + ls = psutil.disk_partitions(all=True) + assert ls + for disk in psutil.disk_partitions(all=True): + check_ntuple(disk) + if not WINDOWS and disk.mountpoint: + try: + os.stat(disk.mountpoint) + except OSError as err: + if GITHUB_ACTIONS and MACOS and err.errno == errno.EIO: + continue + # http://mail.python.org/pipermail/python-dev/ + # 2012-June/120787.html + if err.errno not in {errno.EPERM, errno.EACCES}: + raise + else: + assert os.path.exists(disk.mountpoint), disk + + # --- + + def find_mount_point(path): + path = os.path.abspath(path) + while not os.path.ismount(path): + path = os.path.dirname(path) + return path.lower() + + mount = find_mount_point(__file__) + mounts = [ + x.mountpoint.lower() + for x in psutil.disk_partitions(all=True) + if x.mountpoint + ] + assert mount in mounts + + @pytest.mark.skipif( + LINUX and not os.path.exists('/proc/diskstats'), + reason="/proc/diskstats not available on this linux version", + ) + @pytest.mark.skipif( + CI_TESTING and not psutil.disk_io_counters(), reason="unreliable on CI" + ) # no visible disks + def test_disk_io_counters(self): + def check_ntuple(nt): + assert nt[0] == nt.read_count + assert nt[1] == nt.write_count + assert nt[2] == nt.read_bytes + assert nt[3] == nt.write_bytes + if not (OPENBSD or NETBSD): + assert nt[4] == nt.read_time + assert nt[5] == nt.write_time + if LINUX: + assert nt[6] == nt.read_merged_count + assert nt[7] == nt.write_merged_count + assert nt[8] == nt.busy_time + elif FREEBSD: + assert nt[6] == nt.busy_time + for name in nt._fields: + assert getattr(nt, name) >= 0, nt + + ret = psutil.disk_io_counters(perdisk=False) + assert ret is not None, "no disks on this system?" + check_ntuple(ret) + ret = psutil.disk_io_counters(perdisk=True) + # make sure there are no duplicates + assert len(ret) == len(set(ret)) + for key in ret: + assert key, key + check_ntuple(ret[key]) + + def test_disk_io_counters_no_disks(self): + # Emulate a case where no disks are installed, see: + # https://github.com/giampaolo/psutil/issues/1062 + with mock.patch( + 'psutil._psplatform.disk_io_counters', return_value={} + ) as m: + assert psutil.disk_io_counters(perdisk=False) is None + assert psutil.disk_io_counters(perdisk=True) == {} + assert m.called + + +class TestNetAPIs(PsutilTestCase): + @pytest.mark.skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_net_io_counters(self): + def check_ntuple(nt): + assert nt[0] == nt.bytes_sent + assert nt[1] == nt.bytes_recv + assert nt[2] == nt.packets_sent + assert nt[3] == nt.packets_recv + assert nt[4] == nt.errin + assert nt[5] == nt.errout + assert nt[6] == nt.dropin + assert nt[7] == nt.dropout + assert nt.bytes_sent >= 0, nt + assert nt.bytes_recv >= 0, nt + assert nt.packets_sent >= 0, nt + assert nt.packets_recv >= 0, nt + assert nt.errin >= 0, nt + assert nt.errout >= 0, nt + assert nt.dropin >= 0, nt + assert nt.dropout >= 0, nt + + ret = psutil.net_io_counters(pernic=False) + check_ntuple(ret) + ret = psutil.net_io_counters(pernic=True) + assert ret != [] + for key in ret: + assert key + assert isinstance(key, str) + check_ntuple(ret[key]) + + @pytest.mark.skipif(not HAS_NET_IO_COUNTERS, reason="not supported") + def test_net_io_counters_no_nics(self): + # Emulate a case where no NICs are installed, see: + # https://github.com/giampaolo/psutil/issues/1062 + with mock.patch( + 'psutil._psplatform.net_io_counters', return_value={} + ) as m: + assert psutil.net_io_counters(pernic=False) is None + assert psutil.net_io_counters(pernic=True) == {} + assert m.called + + def test_net_if_addrs(self): + nics = psutil.net_if_addrs() + assert nics, nics + + nic_stats = psutil.net_if_stats() + + # Not reliable on all platforms (net_if_addrs() reports more + # interfaces). + # self.assertEqual(sorted(nics.keys()), + # sorted(psutil.net_io_counters(pernic=True).keys())) + + families = {socket.AF_INET, socket.AF_INET6, psutil.AF_LINK} + for nic, addrs in nics.items(): + assert isinstance(nic, str) + assert len(set(addrs)) == len(addrs) + for addr in addrs: + assert isinstance(addr.family, int) + assert isinstance(addr.address, str) + assert isinstance(addr.netmask, (str, type(None))) + assert isinstance(addr.broadcast, (str, type(None))) + assert addr.family in families + assert isinstance(addr.family, enum.IntEnum) + if nic_stats[nic].isup: + # Do not test binding to addresses of interfaces + # that are down + if addr.family == socket.AF_INET: + with socket.socket(addr.family) as s: + s.bind((addr.address, 0)) + elif addr.family == socket.AF_INET6: + info = socket.getaddrinfo( + addr.address, + 0, + socket.AF_INET6, + socket.SOCK_STREAM, + 0, + socket.AI_PASSIVE, + )[0] + af, socktype, proto, _canonname, sa = info + with socket.socket(af, socktype, proto) as s: + s.bind(sa) + for ip in ( + addr.address, + addr.netmask, + addr.broadcast, + addr.ptp, + ): + if ip is not None: + # TODO: skip AF_INET6 for now because I get: + # AddressValueError: Only hex digits permitted in + # u'c6f3%lxcbr0' in u'fe80::c8e0:fff:fe54:c6f3%lxcbr0' + if addr.family != socket.AF_INET6: + check_net_address(ip, addr.family) + # broadcast and ptp addresses are mutually exclusive + if addr.broadcast: + assert addr.ptp is None + elif addr.ptp: + assert addr.broadcast is None + + # check broadcast address + if ( + addr.broadcast + and addr.netmask + and addr.family in {socket.AF_INET, socket.AF_INET6} + ): + assert addr.broadcast == broadcast_addr(addr) + + if BSD or MACOS or SUNOS: + if hasattr(socket, "AF_LINK"): + assert psutil.AF_LINK == socket.AF_LINK + elif LINUX: + assert psutil.AF_LINK == socket.AF_PACKET + elif WINDOWS: + assert psutil.AF_LINK == -1 + + def test_net_if_addrs_mac_null_bytes(self): + # Simulate that the underlying C function returns an incomplete + # MAC address. psutil is supposed to fill it with null bytes. + # https://github.com/giampaolo/psutil/issues/786 + if POSIX: + ret = [('em1', psutil.AF_LINK, '06:3d:29', None, None, None)] + else: + ret = [('em1', -1, '06-3d-29', None, None, None)] + with mock.patch( + 'psutil._psplatform.net_if_addrs', return_value=ret + ) as m: + addr = psutil.net_if_addrs()['em1'][0] + assert m.called + if POSIX: + assert addr.address == '06:3d:29:00:00:00' + else: + assert addr.address == '06-3d-29-00-00-00' + + def test_net_if_stats(self): + nics = psutil.net_if_stats() + assert nics, nics + all_duplexes = ( + psutil.NIC_DUPLEX_FULL, + psutil.NIC_DUPLEX_HALF, + psutil.NIC_DUPLEX_UNKNOWN, + ) + for name, stats in nics.items(): + assert isinstance(name, str) + isup, duplex, speed, mtu, flags = stats + assert isinstance(isup, bool) + assert duplex in all_duplexes + assert duplex in all_duplexes + assert speed >= 0 + assert mtu >= 0 + assert isinstance(flags, str) + + @pytest.mark.skipif( + not (LINUX or BSD or MACOS), reason="LINUX or BSD or MACOS specific" + ) + def test_net_if_stats_enodev(self): + # See: https://github.com/giampaolo/psutil/issues/1279 + with mock.patch( + 'psutil._psutil_posix.net_if_mtu', + side_effect=OSError(errno.ENODEV, ""), + ) as m: + ret = psutil.net_if_stats() + assert ret == {} + assert m.called + + +class TestSensorsAPIs(PsutilTestCase): + @pytest.mark.skipif(not HAS_SENSORS_TEMPERATURES, reason="not supported") + def test_sensors_temperatures(self): + temps = psutil.sensors_temperatures() + for name, entries in temps.items(): + assert isinstance(name, str) + for entry in entries: + assert isinstance(entry.label, str) + if entry.current is not None: + assert entry.current >= 0 + if entry.high is not None: + assert entry.high >= 0 + if entry.critical is not None: + assert entry.critical >= 0 + + @pytest.mark.skipif(not HAS_SENSORS_TEMPERATURES, reason="not supported") + def test_sensors_temperatures_fahreneit(self): + d = {'coretemp': [('label', 50.0, 60.0, 70.0)]} + with mock.patch( + "psutil._psplatform.sensors_temperatures", return_value=d + ) as m: + temps = psutil.sensors_temperatures(fahrenheit=True)['coretemp'][0] + assert m.called + assert temps.current == 122.0 + assert temps.high == 140.0 + assert temps.critical == 158.0 + + @pytest.mark.skipif(not HAS_SENSORS_BATTERY, reason="not supported") + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_sensors_battery(self): + ret = psutil.sensors_battery() + assert ret.percent >= 0 + assert ret.percent <= 100 + if ret.secsleft not in { + psutil.POWER_TIME_UNKNOWN, + psutil.POWER_TIME_UNLIMITED, + }: + assert ret.secsleft >= 0 + elif ret.secsleft == psutil.POWER_TIME_UNLIMITED: + assert ret.power_plugged + assert isinstance(ret.power_plugged, bool) + + @pytest.mark.skipif(not HAS_SENSORS_FANS, reason="not supported") + def test_sensors_fans(self): + fans = psutil.sensors_fans() + for name, entries in fans.items(): + assert isinstance(name, str) + for entry in entries: + assert isinstance(entry.label, str) + assert isinstance(entry.current, int) + assert entry.current >= 0 diff --git a/lib/python3.12/site-packages/psutil/tests/test_testutils.py b/lib/python3.12/site-packages/psutil/tests/test_testutils.py new file mode 100644 index 0000000000000000000000000000000000000000..6db66e50eb54c8b77222785fb537923ba05f46d3 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_testutils.py @@ -0,0 +1,577 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Tests for testing utils (psutil.tests namespace).""" + +import collections +import errno +import os +import socket +import stat +import subprocess +import textwrap +import unittest +import warnings +from unittest import mock + +import psutil +import psutil.tests +from psutil import FREEBSD +from psutil import NETBSD +from psutil import POSIX +from psutil._common import open_binary +from psutil._common import open_text +from psutil._common import supports_ipv6 +from psutil.tests import CI_TESTING +from psutil.tests import COVERAGE +from psutil.tests import HAS_NET_CONNECTIONS_UNIX +from psutil.tests import HERE +from psutil.tests import PYTHON_EXE +from psutil.tests import PYTHON_EXE_ENV +from psutil.tests import PsutilTestCase +from psutil.tests import TestMemoryLeak +from psutil.tests import bind_socket +from psutil.tests import bind_unix_socket +from psutil.tests import call_until +from psutil.tests import chdir +from psutil.tests import create_sockets +from psutil.tests import fake_pytest +from psutil.tests import filter_proc_net_connections +from psutil.tests import get_free_port +from psutil.tests import is_namedtuple +from psutil.tests import process_namespace +from psutil.tests import pytest +from psutil.tests import reap_children +from psutil.tests import retry +from psutil.tests import retry_on_failure +from psutil.tests import safe_mkdir +from psutil.tests import safe_rmpath +from psutil.tests import system_namespace +from psutil.tests import tcp_socketpair +from psutil.tests import terminate +from psutil.tests import unix_socketpair +from psutil.tests import wait_for_file +from psutil.tests import wait_for_pid + + +# =================================================================== +# --- Unit tests for test utilities. +# =================================================================== + + +class TestRetryDecorator(PsutilTestCase): + @mock.patch('time.sleep') + def test_retry_success(self, sleep): + # Fail 3 times out of 5; make sure the decorated fun returns. + + @retry(retries=5, interval=1, logfun=None) + def foo(): + while queue: + queue.pop() + 1 / 0 # noqa: B018 + return 1 + + queue = list(range(3)) + assert foo() == 1 + assert sleep.call_count == 3 + + @mock.patch('time.sleep') + def test_retry_failure(self, sleep): + # Fail 6 times out of 5; th function is supposed to raise exc. + @retry(retries=5, interval=1, logfun=None) + def foo(): + while queue: + queue.pop() + 1 / 0 # noqa: B018 + return 1 + + queue = list(range(6)) + with pytest.raises(ZeroDivisionError): + foo() + assert sleep.call_count == 5 + + @mock.patch('time.sleep') + def test_exception_arg(self, sleep): + @retry(exception=ValueError, interval=1) + def foo(): + raise TypeError + + with pytest.raises(TypeError): + foo() + assert sleep.call_count == 0 + + @mock.patch('time.sleep') + def test_no_interval_arg(self, sleep): + # if interval is not specified sleep is not supposed to be called + + @retry(retries=5, interval=None, logfun=None) + def foo(): + 1 / 0 # noqa: B018 + + with pytest.raises(ZeroDivisionError): + foo() + assert sleep.call_count == 0 + + @mock.patch('time.sleep') + def test_retries_arg(self, sleep): + @retry(retries=5, interval=1, logfun=None) + def foo(): + 1 / 0 # noqa: B018 + + with pytest.raises(ZeroDivisionError): + foo() + assert sleep.call_count == 5 + + @mock.patch('time.sleep') + def test_retries_and_timeout_args(self, sleep): + with pytest.raises(ValueError): + retry(retries=5, timeout=1) + + +class TestSyncTestUtils(PsutilTestCase): + def test_wait_for_pid(self): + wait_for_pid(os.getpid()) + nopid = max(psutil.pids()) + 99999 + with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): + with pytest.raises(psutil.NoSuchProcess): + wait_for_pid(nopid) + + def test_wait_for_file(self): + testfn = self.get_testfn() + with open(testfn, 'w') as f: + f.write('foo') + wait_for_file(testfn) + assert not os.path.exists(testfn) + + def test_wait_for_file_empty(self): + testfn = self.get_testfn() + with open(testfn, 'w'): + pass + wait_for_file(testfn, empty=True) + assert not os.path.exists(testfn) + + def test_wait_for_file_no_file(self): + testfn = self.get_testfn() + with mock.patch('psutil.tests.retry.__iter__', return_value=iter([0])): + with pytest.raises(OSError): + wait_for_file(testfn) + + def test_wait_for_file_no_delete(self): + testfn = self.get_testfn() + with open(testfn, 'w') as f: + f.write('foo') + wait_for_file(testfn, delete=False) + assert os.path.exists(testfn) + + def test_call_until(self): + call_until(lambda: 1) + # TODO: test for timeout + + +class TestFSTestUtils(PsutilTestCase): + def test_open_text(self): + with open_text(__file__) as f: + assert f.mode == 'r' + + def test_open_binary(self): + with open_binary(__file__) as f: + assert f.mode == 'rb' + + def test_safe_mkdir(self): + testfn = self.get_testfn() + safe_mkdir(testfn) + assert os.path.isdir(testfn) + safe_mkdir(testfn) + assert os.path.isdir(testfn) + + def test_safe_rmpath(self): + # test file is removed + testfn = self.get_testfn() + open(testfn, 'w').close() + safe_rmpath(testfn) + assert not os.path.exists(testfn) + # test no exception if path does not exist + safe_rmpath(testfn) + # test dir is removed + os.mkdir(testfn) + safe_rmpath(testfn) + assert not os.path.exists(testfn) + # test other exceptions are raised + with mock.patch( + 'psutil.tests.os.stat', side_effect=OSError(errno.EINVAL, "") + ) as m: + with pytest.raises(OSError): + safe_rmpath(testfn) + assert m.called + + def test_chdir(self): + testfn = self.get_testfn() + base = os.getcwd() + os.mkdir(testfn) + with chdir(testfn): + assert os.getcwd() == os.path.join(base, testfn) + assert os.getcwd() == base + + +class TestProcessUtils(PsutilTestCase): + def test_reap_children(self): + subp = self.spawn_testproc() + p = psutil.Process(subp.pid) + assert p.is_running() + reap_children() + assert not p.is_running() + assert not psutil.tests._pids_started + assert not psutil.tests._subprocesses_started + + def test_spawn_children_pair(self): + child, grandchild = self.spawn_children_pair() + assert child.pid != grandchild.pid + assert child.is_running() + assert grandchild.is_running() + children = psutil.Process().children() + assert children == [child] + children = psutil.Process().children(recursive=True) + assert len(children) == 2 + assert child in children + assert grandchild in children + assert child.ppid() == os.getpid() + assert grandchild.ppid() == child.pid + + terminate(child) + assert not child.is_running() + assert grandchild.is_running() + + terminate(grandchild) + assert not grandchild.is_running() + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_spawn_zombie(self): + _parent, zombie = self.spawn_zombie() + assert zombie.status() == psutil.STATUS_ZOMBIE + + def test_terminate(self): + # by subprocess.Popen + p = self.spawn_testproc() + terminate(p) + self.assertPidGone(p.pid) + terminate(p) + # by psutil.Process + p = psutil.Process(self.spawn_testproc().pid) + terminate(p) + self.assertPidGone(p.pid) + terminate(p) + # by psutil.Popen + cmd = [ + PYTHON_EXE, + "-c", + "import time; [time.sleep(0.1) for x in range(100)];", + ] + p = psutil.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=PYTHON_EXE_ENV, + ) + terminate(p) + self.assertPidGone(p.pid) + terminate(p) + # by PID + pid = self.spawn_testproc().pid + terminate(pid) + self.assertPidGone(p.pid) + terminate(pid) + # zombie + if POSIX: + parent, zombie = self.spawn_zombie() + terminate(parent) + terminate(zombie) + self.assertPidGone(parent.pid) + self.assertPidGone(zombie.pid) + + +class TestNetUtils(PsutilTestCase): + def bind_socket(self): + port = get_free_port() + with bind_socket(addr=('', port)) as s: + assert s.getsockname()[1] == port + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_bind_unix_socket(self): + name = self.get_testfn() + with bind_unix_socket(name) as sock: + assert sock.family == socket.AF_UNIX + assert sock.type == socket.SOCK_STREAM + assert sock.getsockname() == name + assert os.path.exists(name) + assert stat.S_ISSOCK(os.stat(name).st_mode) + # UDP + name = self.get_testfn() + with bind_unix_socket(name, type=socket.SOCK_DGRAM) as sock: + assert sock.type == socket.SOCK_DGRAM + + def test_tcp_socketpair(self): + addr = ("127.0.0.1", get_free_port()) + server, client = tcp_socketpair(socket.AF_INET, addr=addr) + with server, client: + # Ensure they are connected and the positions are correct. + assert server.getsockname() == addr + assert client.getpeername() == addr + assert client.getsockname() != addr + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + @pytest.mark.skipif( + NETBSD or FREEBSD, reason="/var/run/log UNIX socket opened by default" + ) + def test_unix_socketpair(self): + p = psutil.Process() + num_fds = p.num_fds() + assert not filter_proc_net_connections(p.net_connections(kind='unix')) + name = self.get_testfn() + server, client = unix_socketpair(name) + try: + assert os.path.exists(name) + assert stat.S_ISSOCK(os.stat(name).st_mode) + assert p.num_fds() - num_fds == 2 + assert ( + len( + filter_proc_net_connections(p.net_connections(kind='unix')) + ) + == 2 + ) + assert server.getsockname() == name + assert client.getpeername() == name + finally: + client.close() + server.close() + + def test_create_sockets(self): + with create_sockets() as socks: + fams = collections.defaultdict(int) + types = collections.defaultdict(int) + for s in socks: + fams[s.family] += 1 + # work around http://bugs.python.org/issue30204 + types[s.getsockopt(socket.SOL_SOCKET, socket.SO_TYPE)] += 1 + assert fams[socket.AF_INET] >= 2 + if supports_ipv6(): + assert fams[socket.AF_INET6] >= 2 + if POSIX and HAS_NET_CONNECTIONS_UNIX: + assert fams[socket.AF_UNIX] >= 2 + assert types[socket.SOCK_STREAM] >= 2 + assert types[socket.SOCK_DGRAM] >= 2 + + +@pytest.mark.xdist_group(name="serial") +class TestMemLeakClass(TestMemoryLeak): + @retry_on_failure() + def test_times(self): + def fun(): + cnt['cnt'] += 1 + + cnt = {'cnt': 0} + self.execute(fun, times=10, warmup_times=15) + assert cnt['cnt'] == 26 + + def test_param_err(self): + with pytest.raises(ValueError): + self.execute(lambda: 0, times=0) + with pytest.raises(ValueError): + self.execute(lambda: 0, times=-1) + with pytest.raises(ValueError): + self.execute(lambda: 0, warmup_times=-1) + with pytest.raises(ValueError): + self.execute(lambda: 0, tolerance=-1) + with pytest.raises(ValueError): + self.execute(lambda: 0, retries=-1) + + @retry_on_failure() + @pytest.mark.skipif(CI_TESTING, reason="skipped on CI") + @pytest.mark.skipif(COVERAGE, reason="skipped during test coverage") + def test_leak_mem(self): + ls = [] + + def fun(ls=ls): + ls.append("x" * 248 * 1024) + + try: + # will consume around 60M in total + with pytest.raises(AssertionError, match="extra-mem"): + self.execute(fun, times=100) + finally: + del ls + + def test_unclosed_files(self): + def fun(): + f = open(__file__) # noqa: SIM115 + self.addCleanup(f.close) + box.append(f) + + box = [] + kind = "fd" if POSIX else "handle" + with pytest.raises(AssertionError, match="unclosed " + kind): + self.execute(fun) + + def test_tolerance(self): + def fun(): + ls.append("x" * 24 * 1024) + + ls = [] + times = 100 + self.execute( + fun, times=times, warmup_times=0, tolerance=200 * 1024 * 1024 + ) + assert len(ls) == times + 1 + + def test_execute_w_exc(self): + def fun_1(): + 1 / 0 # noqa: B018 + + self.execute_w_exc(ZeroDivisionError, fun_1) + with pytest.raises(ZeroDivisionError): + self.execute_w_exc(OSError, fun_1) + + def fun_2(): + pass + + with pytest.raises(AssertionError): + self.execute_w_exc(ZeroDivisionError, fun_2) + + +class TestFakePytest(PsutilTestCase): + def run_test_class(self, klass): + suite = unittest.TestSuite() + suite.addTest(klass) + runner = unittest.TextTestRunner() + result = runner.run(suite) + return result + + def test_raises(self): + with fake_pytest.raises(ZeroDivisionError) as cm: + 1 / 0 # noqa: B018 + assert isinstance(cm.value, ZeroDivisionError) + + with fake_pytest.raises(ValueError, match="foo") as cm: + raise ValueError("foo") + + try: + with fake_pytest.raises(ValueError, match="foo") as cm: + raise ValueError("bar") + except AssertionError as err: + assert str(err) == '"foo" does not match "bar"' + else: + raise self.fail("exception not raised") + + def test_mark(self): + @fake_pytest.mark.xdist_group(name="serial") + def foo(): + return 1 + + assert foo() == 1 + + @fake_pytest.mark.xdist_group(name="serial") + class Foo: + def bar(self): + return 1 + + assert Foo().bar() == 1 + + def test_skipif(self): + class TestCase(unittest.TestCase): + @fake_pytest.mark.skipif(True, reason="reason") + def foo(self): + assert 1 == 1 # noqa: PLR0133 + + result = self.run_test_class(TestCase("foo")) + assert result.wasSuccessful() + assert len(result.skipped) == 1 + assert result.skipped[0][1] == "reason" + + class TestCase(unittest.TestCase): + @fake_pytest.mark.skipif(False, reason="reason") + def foo(self): + assert 1 == 1 # noqa: PLR0133 + + result = self.run_test_class(TestCase("foo")) + assert result.wasSuccessful() + assert len(result.skipped) == 0 + + def test_skip(self): + class TestCase(unittest.TestCase): + def foo(self): + fake_pytest.skip("reason") + assert 1 == 0 # noqa: PLR0133 + + result = self.run_test_class(TestCase("foo")) + assert result.wasSuccessful() + assert len(result.skipped) == 1 + assert result.skipped[0][1] == "reason" + + def test_main(self): + tmpdir = self.get_testfn(dir=HERE) + os.mkdir(tmpdir) + with open(os.path.join(tmpdir, "__init__.py"), "w"): + pass + with open(os.path.join(tmpdir, "test_file.py"), "w") as f: + f.write(textwrap.dedent("""\ + import unittest + + class TestCase(unittest.TestCase): + def test_passed(self): + pass + """).lstrip()) + with mock.patch.object(psutil.tests, "HERE", tmpdir): + with self.assertWarnsRegex( + UserWarning, "Fake pytest module was used" + ): + suite = fake_pytest.main() + assert suite.countTestCases() == 1 + + def test_warns(self): + # success + with fake_pytest.warns(UserWarning): + warnings.warn("foo", UserWarning, stacklevel=1) + + # failure + try: + with fake_pytest.warns(UserWarning): + warnings.warn("foo", DeprecationWarning, stacklevel=1) + except AssertionError: + pass + else: + raise self.fail("exception not raised") + + # match success + with fake_pytest.warns(UserWarning, match="foo"): + warnings.warn("foo", UserWarning, stacklevel=1) + + # match failure + try: + with fake_pytest.warns(UserWarning, match="foo"): + warnings.warn("bar", UserWarning, stacklevel=1) + except AssertionError: + pass + else: + raise self.fail("exception not raised") + + +class TestTestingUtils(PsutilTestCase): + def test_process_namespace(self): + p = psutil.Process() + ns = process_namespace(p) + ns.test() + fun = next(x for x in ns.iter(ns.getters) if x[1] == 'ppid')[0] + assert fun() == p.ppid() + + def test_system_namespace(self): + ns = system_namespace() + fun = next(x for x in ns.iter(ns.getters) if x[1] == 'net_if_addrs')[0] + assert fun() == psutil.net_if_addrs() + + +class TestOtherUtils(PsutilTestCase): + def test_is_namedtuple(self): + assert is_namedtuple(collections.namedtuple('foo', 'a b c')(1, 2, 3)) + assert not is_namedtuple(tuple()) diff --git a/lib/python3.12/site-packages/psutil/tests/test_unicode.py b/lib/python3.12/site-packages/psutil/tests/test_unicode.py new file mode 100644 index 0000000000000000000000000000000000000000..d8a8c4bfc55e7f077a6f27ca2b42dab183ba88ed --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_unicode.py @@ -0,0 +1,313 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Notes about unicode handling in psutil +======================================. + +Starting from version 5.3.0 psutil adds unicode support, see: +https://github.com/giampaolo/psutil/issues/1040 +The notes below apply to *any* API returning a string such as +process exe(), cwd() or username(): + +* all strings are encoded by using the OS filesystem encoding + (sys.getfilesystemencoding()) which varies depending on the platform + (e.g. "UTF-8" on macOS, "mbcs" on Win) +* no API call is supposed to crash with UnicodeDecodeError +* instead, in case of badly encoded data returned by the OS, the + following error handlers are used to replace the corrupted characters in + the string: + * sys.getfilesystemencodeerrors() or "surrogatescape" on POSIX and + "replace" on Windows. + +For a detailed explanation of how psutil handles unicode see #1040. + +Tests +===== + +List of APIs returning or dealing with a string: +('not tested' means they are not tested to deal with non-ASCII strings): + +* Process.cmdline() +* Process.cwd() +* Process.environ() +* Process.exe() +* Process.memory_maps() +* Process.name() +* Process.net_connections('unix') +* Process.open_files() +* Process.username() (not tested) + +* disk_io_counters() (not tested) +* disk_partitions() (not tested) +* disk_usage(str) +* net_connections('unix') +* net_if_addrs() (not tested) +* net_if_stats() (not tested) +* net_io_counters() (not tested) +* sensors_fans() (not tested) +* sensors_temperatures() (not tested) +* users() (not tested) + +* WindowsService.binpath() (not tested) +* WindowsService.description() (not tested) +* WindowsService.display_name() (not tested) +* WindowsService.name() (not tested) +* WindowsService.status() (not tested) +* WindowsService.username() (not tested) + +In here we create a unicode path with a funky non-ASCII name and (where +possible) make psutil return it back (e.g. on name(), exe(), open_files(), +etc.) and make sure that: + +* psutil never crashes with UnicodeDecodeError +* the returned path matches +""" + +import os +import shutil +import warnings +from contextlib import closing + +import psutil +from psutil import BSD +from psutil import POSIX +from psutil import WINDOWS +from psutil.tests import ASCII_FS +from psutil.tests import CI_TESTING +from psutil.tests import HAS_ENVIRON +from psutil.tests import HAS_MEMORY_MAPS +from psutil.tests import HAS_NET_CONNECTIONS_UNIX +from psutil.tests import INVALID_UNICODE_SUFFIX +from psutil.tests import PYPY +from psutil.tests import TESTFN_PREFIX +from psutil.tests import UNICODE_SUFFIX +from psutil.tests import PsutilTestCase +from psutil.tests import bind_unix_socket +from psutil.tests import chdir +from psutil.tests import copyload_shared_lib +from psutil.tests import create_py_exe +from psutil.tests import get_testfn +from psutil.tests import pytest +from psutil.tests import safe_mkdir +from psutil.tests import safe_rmpath +from psutil.tests import skip_on_access_denied +from psutil.tests import spawn_testproc +from psutil.tests import terminate + + +def try_unicode(suffix): + """Return True if both the fs and the subprocess module can + deal with a unicode file name. + """ + sproc = None + testfn = get_testfn(suffix=suffix) + try: + safe_rmpath(testfn) + create_py_exe(testfn) + sproc = spawn_testproc(cmd=[testfn]) + shutil.copyfile(testfn, testfn + '-2') + safe_rmpath(testfn + '-2') + except (UnicodeEncodeError, OSError): + return False + else: + return True + finally: + if sproc is not None: + terminate(sproc) + safe_rmpath(testfn) + + +# =================================================================== +# FS APIs +# =================================================================== + + +class BaseUnicodeTest(PsutilTestCase): + funky_suffix = None + + @classmethod + def setUpClass(cls): + super().setUpClass() + cls.skip_tests = False + cls.funky_name = None + if cls.funky_suffix is not None: + if not try_unicode(cls.funky_suffix): + cls.skip_tests = True + else: + cls.funky_name = get_testfn(suffix=cls.funky_suffix) + create_py_exe(cls.funky_name) + + def setUp(self): + super().setUp() + if self.skip_tests: + raise pytest.skip("can't handle unicode str") + + +@pytest.mark.xdist_group(name="serial") +@pytest.mark.skipif(ASCII_FS, reason="ASCII fs") +class TestFSAPIs(BaseUnicodeTest): + """Test FS APIs with a funky, valid, UTF8 path name.""" + + funky_suffix = UNICODE_SUFFIX + + def expect_exact_path_match(self): + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + return self.funky_name in os.listdir(".") + + # --- + + def test_proc_exe(self): + cmd = [ + self.funky_name, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] + subp = self.spawn_testproc(cmd) + p = psutil.Process(subp.pid) + exe = p.exe() + assert isinstance(exe, str) + if self.expect_exact_path_match(): + assert os.path.normcase(exe) == os.path.normcase(self.funky_name) + + def test_proc_name(self): + cmd = [ + self.funky_name, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] + subp = self.spawn_testproc(cmd) + name = psutil.Process(subp.pid).name() + assert isinstance(name, str) + if self.expect_exact_path_match(): + assert name == os.path.basename(self.funky_name) + + def test_proc_cmdline(self): + cmd = [ + self.funky_name, + "-c", + "import time; [time.sleep(0.1) for x in range(100)]", + ] + subp = self.spawn_testproc(cmd) + p = psutil.Process(subp.pid) + cmdline = p.cmdline() + for part in cmdline: + assert isinstance(part, str) + if self.expect_exact_path_match(): + assert cmdline == cmd + + def test_proc_cwd(self): + dname = self.funky_name + "2" + self.addCleanup(safe_rmpath, dname) + safe_mkdir(dname) + with chdir(dname): + p = psutil.Process() + cwd = p.cwd() + assert isinstance(p.cwd(), str) + if self.expect_exact_path_match(): + assert cwd == dname + + @pytest.mark.skipif(PYPY and WINDOWS, reason="fails on PYPY + WINDOWS") + def test_proc_open_files(self): + p = psutil.Process() + start = set(p.open_files()) + with open(self.funky_name, 'rb'): + new = set(p.open_files()) + path = (new - start).pop().path + assert isinstance(path, str) + if BSD and not path: + # XXX - see https://github.com/giampaolo/psutil/issues/595 + raise pytest.skip("open_files on BSD is broken") + if self.expect_exact_path_match(): + assert os.path.normcase(path) == os.path.normcase(self.funky_name) + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + def test_proc_net_connections(self): + name = self.get_testfn(suffix=self.funky_suffix) + sock = bind_unix_socket(name) + with closing(sock): + conn = psutil.Process().net_connections('unix')[0] + assert isinstance(conn.laddr, str) + assert conn.laddr == name + + @pytest.mark.skipif(not POSIX, reason="POSIX only") + @pytest.mark.skipif( + not HAS_NET_CONNECTIONS_UNIX, reason="can't list UNIX sockets" + ) + @skip_on_access_denied() + def test_net_connections(self): + def find_sock(cons): + for conn in cons: + if os.path.basename(conn.laddr).startswith(TESTFN_PREFIX): + return conn + raise ValueError("connection not found") + + name = self.get_testfn(suffix=self.funky_suffix) + sock = bind_unix_socket(name) + with closing(sock): + cons = psutil.net_connections(kind='unix') + conn = find_sock(cons) + assert isinstance(conn.laddr, str) + assert conn.laddr == name + + def test_disk_usage(self): + dname = self.funky_name + "2" + self.addCleanup(safe_rmpath, dname) + safe_mkdir(dname) + psutil.disk_usage(dname) + + @pytest.mark.skipif(not HAS_MEMORY_MAPS, reason="not supported") + @pytest.mark.skipif(PYPY, reason="unstable on PYPY") + def test_memory_maps(self): + with copyload_shared_lib(suffix=self.funky_suffix) as funky_path: + + def normpath(p): + return os.path.realpath(os.path.normcase(p)) + + libpaths = [ + normpath(x.path) for x in psutil.Process().memory_maps() + ] + # ...just to have a clearer msg in case of failure + libpaths = [x for x in libpaths if TESTFN_PREFIX in x] + assert normpath(funky_path) in libpaths + for path in libpaths: + assert isinstance(path, str) + + +@pytest.mark.skipif(CI_TESTING, reason="unreliable on CI") +class TestFSAPIsWithInvalidPath(TestFSAPIs): + """Test FS APIs with a funky, invalid path name.""" + + funky_suffix = INVALID_UNICODE_SUFFIX + + def expect_exact_path_match(self): + return True + + +# =================================================================== +# Non fs APIs +# =================================================================== + + +class TestNonFSAPIS(BaseUnicodeTest): + """Unicode tests for non fs-related APIs.""" + + funky_suffix = UNICODE_SUFFIX + + @pytest.mark.skipif(not HAS_ENVIRON, reason="not supported") + @pytest.mark.skipif(PYPY and WINDOWS, reason="segfaults on PYPY + WINDOWS") + def test_proc_environ(self): + # Note: differently from others, this test does not deal + # with fs paths. + env = os.environ.copy() + env['FUNNY_ARG'] = self.funky_suffix + sproc = self.spawn_testproc(env=env) + p = psutil.Process(sproc.pid) + env = p.environ() + for k, v in env.items(): + assert isinstance(k, str) + assert isinstance(v, str) + assert env['FUNNY_ARG'] == self.funky_suffix diff --git a/lib/python3.12/site-packages/psutil/tests/test_windows.py b/lib/python3.12/site-packages/psutil/tests/test_windows.py new file mode 100644 index 0000000000000000000000000000000000000000..c5c536b468d44dfbbe44f4d5cfa6f443913cca57 --- /dev/null +++ b/lib/python3.12/site-packages/psutil/tests/test_windows.py @@ -0,0 +1,914 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2009, Giampaolo Rodola'. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Windows specific tests.""" + +import datetime +import glob +import os +import platform +import re +import shutil +import signal +import subprocess +import sys +import time +import warnings +from unittest import mock + +import psutil +from psutil import WINDOWS +from psutil.tests import GITHUB_ACTIONS +from psutil.tests import HAS_BATTERY +from psutil.tests import IS_64BIT +from psutil.tests import PYPY +from psutil.tests import TOLERANCE_DISK_USAGE +from psutil.tests import TOLERANCE_SYS_MEM +from psutil.tests import PsutilTestCase +from psutil.tests import pytest +from psutil.tests import retry_on_failure +from psutil.tests import sh +from psutil.tests import spawn_testproc +from psutil.tests import terminate + + +if WINDOWS and not PYPY: + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + import win32api # requires "pip install pywin32" + import win32con + import win32process + import wmi # requires "pip install wmi" / "make install-pydeps-test" + +if WINDOWS: + from psutil._pswindows import convert_oserror + + +cext = psutil._psplatform.cext + + +@pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") +@pytest.mark.skipif(PYPY, reason="pywin32 not available on PYPY") +class WindowsTestCase(PsutilTestCase): + pass + + +def powershell(cmd): + """Currently not used, but available just in case. Usage: + + >>> powershell( + "Get-CIMInstance Win32_PageFileUsage | Select AllocatedBaseSize") + """ + if not shutil.which("powershell.exe"): + raise pytest.skip("powershell.exe not available") + cmdline = ( + "powershell.exe -ExecutionPolicy Bypass -NoLogo -NonInteractive " + f"-NoProfile -WindowStyle Hidden -Command \"{cmd}\"" # noqa: Q003 + ) + return sh(cmdline) + + +def wmic(path, what, converter=int): + """Currently not used, but available just in case. Usage: + + >>> wmic("Win32_OperatingSystem", "FreePhysicalMemory") + 2134124534 + """ + out = sh(f"wmic path {path} get {what}").strip() + data = "".join(out.splitlines()[1:]).strip() # get rid of the header + if converter is not None: + if "," in what: + return tuple(converter(x) for x in data.split()) + else: + return converter(data) + else: + return data + + +# =================================================================== +# System APIs +# =================================================================== + + +class TestCpuAPIs(WindowsTestCase): + @pytest.mark.skipif( + 'NUMBER_OF_PROCESSORS' not in os.environ, + reason="NUMBER_OF_PROCESSORS env var is not available", + ) + def test_cpu_count_vs_NUMBER_OF_PROCESSORS(self): + # Will likely fail on many-cores systems: + # https://stackoverflow.com/questions/31209256 + num_cpus = int(os.environ['NUMBER_OF_PROCESSORS']) + assert num_cpus == psutil.cpu_count() + + def test_cpu_count_vs_GetSystemInfo(self): + # Will likely fail on many-cores systems: + # https://stackoverflow.com/questions/31209256 + sys_value = win32api.GetSystemInfo()[5] + psutil_value = psutil.cpu_count() + assert sys_value == psutil_value + + def test_cpu_count_logical_vs_wmi(self): + w = wmi.WMI() + procs = sum( + proc.NumberOfLogicalProcessors for proc in w.Win32_Processor() + ) + assert psutil.cpu_count() == procs + + def test_cpu_count_cores_vs_wmi(self): + w = wmi.WMI() + cores = sum(proc.NumberOfCores for proc in w.Win32_Processor()) + assert psutil.cpu_count(logical=False) == cores + + def test_cpu_count_vs_cpu_times(self): + assert psutil.cpu_count() == len(psutil.cpu_times(percpu=True)) + + def test_cpu_freq(self): + w = wmi.WMI() + proc = w.Win32_Processor()[0] + assert proc.CurrentClockSpeed == psutil.cpu_freq().current + assert proc.MaxClockSpeed == psutil.cpu_freq().max + + +class TestSystemAPIs(WindowsTestCase): + def test_nic_names(self): + out = sh('ipconfig /all') + nics = psutil.net_io_counters(pernic=True).keys() + for nic in nics: + if "pseudo-interface" in nic.replace(' ', '-').lower(): + continue + if nic not in out: + raise self.fail( + f"{nic!r} nic wasn't found in 'ipconfig /all' output" + ) + + def test_total_phymem(self): + w = wmi.WMI().Win32_ComputerSystem()[0] + assert int(w.TotalPhysicalMemory) == psutil.virtual_memory().total + + def test_free_phymem(self): + w = wmi.WMI().Win32_PerfRawData_PerfOS_Memory()[0] + assert ( + abs(int(w.AvailableBytes) - psutil.virtual_memory().free) + < TOLERANCE_SYS_MEM + ) + + def test_total_swapmem(self): + w = wmi.WMI().Win32_PerfRawData_PerfOS_Memory()[0] + assert ( + int(w.CommitLimit) - psutil.virtual_memory().total + == psutil.swap_memory().total + ) + if psutil.swap_memory().total == 0: + assert psutil.swap_memory().free == 0 + assert psutil.swap_memory().used == 0 + + def test_percent_swapmem(self): + if psutil.swap_memory().total > 0: + w = wmi.WMI().Win32_PerfRawData_PerfOS_PagingFile(Name="_Total")[0] + # calculate swap usage to percent + percentSwap = int(w.PercentUsage) * 100 / int(w.PercentUsage_Base) + # exact percent may change but should be reasonable + # assert within +/- 5% and between 0 and 100% + assert psutil.swap_memory().percent >= 0 + assert abs(psutil.swap_memory().percent - percentSwap) < 5 + assert psutil.swap_memory().percent <= 100 + + # @pytest.mark.skipif(wmi is None, reason="wmi module is not installed") + # def test__UPTIME(self): + # # _UPTIME constant is not public but it is used internally + # # as value to return for pid 0 creation time. + # # WMI behaves the same. + # w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + # p = psutil.Process(0) + # wmic_create = str(w.CreationDate.split('.')[0]) + # psutil_create = time.strftime("%Y%m%d%H%M%S", + # time.localtime(p.create_time())) + + # Note: this test is not very reliable + @retry_on_failure() + def test_pids(self): + # Note: this test might fail if the OS is starting/killing + # other processes in the meantime + w = wmi.WMI().Win32_Process() + wmi_pids = {x.ProcessId for x in w} + psutil_pids = set(psutil.pids()) + assert wmi_pids == psutil_pids + + @retry_on_failure() + def test_disks(self): + ps_parts = psutil.disk_partitions(all=True) + wmi_parts = wmi.WMI().Win32_LogicalDisk() + for ps_part in ps_parts: + for wmi_part in wmi_parts: + if ps_part.device.replace('\\', '') == wmi_part.DeviceID: + if not ps_part.mountpoint: + # this is usually a CD-ROM with no disk inserted + break + if 'cdrom' in ps_part.opts: + break + if ps_part.mountpoint.startswith('A:'): + break # floppy + try: + usage = psutil.disk_usage(ps_part.mountpoint) + except FileNotFoundError: + # usually this is the floppy + break + assert usage.total == int(wmi_part.Size) + wmi_free = int(wmi_part.FreeSpace) + assert usage.free == wmi_free + # 10 MB tolerance + if abs(usage.free - wmi_free) > 10 * 1024 * 1024: + raise self.fail(f"psutil={usage.free}, wmi={wmi_free}") + break + else: + raise self.fail(f"can't find partition {ps_part!r}") + + @retry_on_failure() + def test_disk_usage(self): + for disk in psutil.disk_partitions(): + if 'cdrom' in disk.opts: + continue + sys_value = win32api.GetDiskFreeSpaceEx(disk.mountpoint) + psutil_value = psutil.disk_usage(disk.mountpoint) + assert abs(sys_value[0] - psutil_value.free) < TOLERANCE_DISK_USAGE + assert ( + abs(sys_value[1] - psutil_value.total) < TOLERANCE_DISK_USAGE + ) + assert psutil_value.used == psutil_value.total - psutil_value.free + + def test_disk_partitions(self): + sys_value = [ + x + '\\' + for x in win32api.GetLogicalDriveStrings().split("\\\x00") + if x and not x.startswith('A:') + ] + psutil_value = [ + x.mountpoint + for x in psutil.disk_partitions(all=True) + if not x.mountpoint.startswith('A:') + ] + assert sys_value == psutil_value + + def test_net_if_stats(self): + ps_names = set(cext.net_if_stats()) + wmi_adapters = wmi.WMI().Win32_NetworkAdapter() + wmi_names = set() + for wmi_adapter in wmi_adapters: + wmi_names.add(wmi_adapter.Name) + wmi_names.add(wmi_adapter.NetConnectionID) + assert ( + ps_names & wmi_names + ), f"no common entries in {ps_names}, {wmi_names}" + + def test_boot_time(self): + wmi_os = wmi.WMI().Win32_OperatingSystem() + wmi_btime_str = wmi_os[0].LastBootUpTime.split('.')[0] + wmi_btime_dt = datetime.datetime.strptime( + wmi_btime_str, "%Y%m%d%H%M%S" + ) + psutil_dt = datetime.datetime.fromtimestamp(psutil.boot_time()) + diff = abs((wmi_btime_dt - psutil_dt).total_seconds()) + assert diff <= 5 + + def test_boot_time_fluctuation(self): + # https://github.com/giampaolo/psutil/issues/1007 + with mock.patch('psutil._pswindows.cext.boot_time', return_value=5): + assert psutil.boot_time() == 5 + with mock.patch('psutil._pswindows.cext.boot_time', return_value=4): + assert psutil.boot_time() == 5 + with mock.patch('psutil._pswindows.cext.boot_time', return_value=6): + assert psutil.boot_time() == 5 + with mock.patch('psutil._pswindows.cext.boot_time', return_value=333): + assert psutil.boot_time() == 333 + + +# =================================================================== +# sensors_battery() +# =================================================================== + + +class TestSensorsBattery(WindowsTestCase): + def test_has_battery(self): + if win32api.GetPwrCapabilities()['SystemBatteriesPresent']: + assert psutil.sensors_battery() is not None + else: + assert psutil.sensors_battery() is None + + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_percent(self): + w = wmi.WMI() + battery_wmi = w.query('select * from Win32_Battery')[0] + battery_psutil = psutil.sensors_battery() + assert ( + abs(battery_psutil.percent - battery_wmi.EstimatedChargeRemaining) + < 1 + ) + + @pytest.mark.skipif(not HAS_BATTERY, reason="no battery") + def test_power_plugged(self): + w = wmi.WMI() + battery_wmi = w.query('select * from Win32_Battery')[0] + battery_psutil = psutil.sensors_battery() + # Status codes: + # https://msdn.microsoft.com/en-us/library/aa394074(v=vs.85).aspx + assert battery_psutil.power_plugged == (battery_wmi.BatteryStatus == 2) + + def test_emulate_no_battery(self): + with mock.patch( + "psutil._pswindows.cext.sensors_battery", + return_value=(0, 128, 0, 0), + ) as m: + assert psutil.sensors_battery() is None + assert m.called + + def test_emulate_power_connected(self): + with mock.patch( + "psutil._pswindows.cext.sensors_battery", return_value=(1, 0, 0, 0) + ) as m: + assert ( + psutil.sensors_battery().secsleft + == psutil.POWER_TIME_UNLIMITED + ) + assert m.called + + def test_emulate_power_charging(self): + with mock.patch( + "psutil._pswindows.cext.sensors_battery", return_value=(0, 8, 0, 0) + ) as m: + assert ( + psutil.sensors_battery().secsleft + == psutil.POWER_TIME_UNLIMITED + ) + assert m.called + + def test_emulate_secs_left_unknown(self): + with mock.patch( + "psutil._pswindows.cext.sensors_battery", + return_value=(0, 0, 0, -1), + ) as m: + assert ( + psutil.sensors_battery().secsleft == psutil.POWER_TIME_UNKNOWN + ) + assert m.called + + +# =================================================================== +# Process APIs +# =================================================================== + + +class TestProcess(WindowsTestCase): + @classmethod + def setUpClass(cls): + cls.pid = spawn_testproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + def test_issue_24(self): + p = psutil.Process(0) + with pytest.raises(psutil.AccessDenied): + p.kill() + + def test_special_pid(self): + p = psutil.Process(4) + assert p.name() == 'System' + # use __str__ to access all common Process properties to check + # that nothing strange happens + str(p) + p.username() + assert p.create_time() >= 0.0 + try: + rss, _vms = p.memory_info()[:2] + except psutil.AccessDenied: + # expected on Windows Vista and Windows 7 + if platform.uname()[1] not in {'vista', 'win-7', 'win7'}: + raise + else: + assert rss > 0 + + def test_send_signal(self): + p = psutil.Process(self.pid) + with pytest.raises(ValueError): + p.send_signal(signal.SIGINT) + + def test_num_handles_increment(self): + p = psutil.Process(os.getpid()) + before = p.num_handles() + handle = win32api.OpenProcess( + win32con.PROCESS_QUERY_INFORMATION, win32con.FALSE, os.getpid() + ) + after = p.num_handles() + assert after == before + 1 + win32api.CloseHandle(handle) + assert p.num_handles() == before + + def test_ctrl_signals(self): + p = psutil.Process(self.spawn_testproc().pid) + p.send_signal(signal.CTRL_C_EVENT) + p.send_signal(signal.CTRL_BREAK_EVENT) + p.kill() + p.wait() + with pytest.raises(psutil.NoSuchProcess): + p.send_signal(signal.CTRL_C_EVENT) + with pytest.raises(psutil.NoSuchProcess): + p.send_signal(signal.CTRL_BREAK_EVENT) + + def test_username(self): + name = win32api.GetUserNameEx(win32con.NameSamCompatible) + if name.endswith('$'): + # When running as a service account (most likely to be + # NetworkService), these user name calculations don't produce the + # same result, causing the test to fail. + raise pytest.skip('running as service account') + assert psutil.Process().username() == name + + def test_cmdline(self): + sys_value = re.sub(r"[ ]+", " ", win32api.GetCommandLine()).strip() + psutil_value = ' '.join(psutil.Process().cmdline()) + if sys_value[0] == '"' != psutil_value[0]: + # The PyWin32 command line may retain quotes around argv[0] if they + # were used unnecessarily, while psutil will omit them. So remove + # the first 2 quotes from sys_value if not in psutil_value. + # A path to an executable will not contain quotes, so this is safe. + sys_value = sys_value.replace('"', '', 2) + assert sys_value == psutil_value + + # XXX - occasional failures + + # def test_cpu_times(self): + # handle = win32api.OpenProcess(win32con.PROCESS_QUERY_INFORMATION, + # win32con.FALSE, os.getpid()) + # self.addCleanup(win32api.CloseHandle, handle) + # sys_value = win32process.GetProcessTimes(handle) + # psutil_value = psutil.Process().cpu_times() + # self.assertAlmostEqual( + # psutil_value.user, sys_value['UserTime'] / 10000000.0, + # delta=0.2) + # self.assertAlmostEqual( + # psutil_value.user, sys_value['KernelTime'] / 10000000.0, + # delta=0.2) + + def test_nice(self): + handle = win32api.OpenProcess( + win32con.PROCESS_QUERY_INFORMATION, win32con.FALSE, os.getpid() + ) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = win32process.GetPriorityClass(handle) + psutil_value = psutil.Process().nice() + assert psutil_value == sys_value + + def test_memory_info(self): + handle = win32api.OpenProcess( + win32con.PROCESS_QUERY_INFORMATION, win32con.FALSE, self.pid + ) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = win32process.GetProcessMemoryInfo(handle) + psutil_value = psutil.Process(self.pid).memory_info() + assert sys_value['PeakWorkingSetSize'] == psutil_value.peak_wset + assert sys_value['WorkingSetSize'] == psutil_value.wset + assert ( + sys_value['QuotaPeakPagedPoolUsage'] + == psutil_value.peak_paged_pool + ) + assert sys_value['QuotaPagedPoolUsage'] == psutil_value.paged_pool + assert ( + sys_value['QuotaPeakNonPagedPoolUsage'] + == psutil_value.peak_nonpaged_pool + ) + assert ( + sys_value['QuotaNonPagedPoolUsage'] == psutil_value.nonpaged_pool + ) + assert sys_value['PagefileUsage'] == psutil_value.pagefile + assert sys_value['PeakPagefileUsage'] == psutil_value.peak_pagefile + + assert psutil_value.rss == psutil_value.wset + assert psutil_value.vms == psutil_value.pagefile + + def test_wait(self): + handle = win32api.OpenProcess( + win32con.PROCESS_QUERY_INFORMATION, win32con.FALSE, self.pid + ) + self.addCleanup(win32api.CloseHandle, handle) + p = psutil.Process(self.pid) + p.terminate() + psutil_value = p.wait() + sys_value = win32process.GetExitCodeProcess(handle) + assert psutil_value == sys_value + + def test_cpu_affinity(self): + def from_bitmask(x): + return [i for i in range(64) if (1 << i) & x] + + handle = win32api.OpenProcess( + win32con.PROCESS_QUERY_INFORMATION, win32con.FALSE, self.pid + ) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = from_bitmask( + win32process.GetProcessAffinityMask(handle)[0] + ) + psutil_value = psutil.Process(self.pid).cpu_affinity() + assert psutil_value == sys_value + + def test_io_counters(self): + handle = win32api.OpenProcess( + win32con.PROCESS_QUERY_INFORMATION, win32con.FALSE, os.getpid() + ) + self.addCleanup(win32api.CloseHandle, handle) + sys_value = win32process.GetProcessIoCounters(handle) + psutil_value = psutil.Process().io_counters() + assert psutil_value.read_count == sys_value['ReadOperationCount'] + assert psutil_value.write_count == sys_value['WriteOperationCount'] + assert psutil_value.read_bytes == sys_value['ReadTransferCount'] + assert psutil_value.write_bytes == sys_value['WriteTransferCount'] + assert psutil_value.other_count == sys_value['OtherOperationCount'] + assert psutil_value.other_bytes == sys_value['OtherTransferCount'] + + def test_num_handles(self): + import ctypes + import ctypes.wintypes + + PROCESS_QUERY_INFORMATION = 0x400 + handle = ctypes.windll.kernel32.OpenProcess( + PROCESS_QUERY_INFORMATION, 0, self.pid + ) + self.addCleanup(ctypes.windll.kernel32.CloseHandle, handle) + + hndcnt = ctypes.wintypes.DWORD() + ctypes.windll.kernel32.GetProcessHandleCount( + handle, ctypes.byref(hndcnt) + ) + sys_value = hndcnt.value + psutil_value = psutil.Process(self.pid).num_handles() + assert psutil_value == sys_value + + def test_error_partial_copy(self): + # https://github.com/giampaolo/psutil/issues/875 + exc = OSError() + exc.winerror = 299 + with mock.patch("psutil._psplatform.cext.proc_cwd", side_effect=exc): + with mock.patch("time.sleep") as m: + p = psutil.Process() + with pytest.raises(psutil.AccessDenied): + p.cwd() + assert m.call_count >= 5 + + def test_exe(self): + # NtQuerySystemInformation succeeds if process is gone. Make sure + # it raises NSP for a non existent pid. + pid = psutil.pids()[-1] + 99999 + proc = psutil._psplatform.Process(pid) + with pytest.raises(psutil.NoSuchProcess): + proc.exe() + + +class TestProcessWMI(WindowsTestCase): + """Compare Process API results with WMI.""" + + @classmethod + def setUpClass(cls): + cls.pid = spawn_testproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + def test_name(self): + w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + p = psutil.Process(self.pid) + assert p.name() == w.Caption + + # This fail on github because using virtualenv for test environment + @pytest.mark.skipif( + GITHUB_ACTIONS, reason="unreliable path on GITHUB_ACTIONS" + ) + def test_exe(self): + w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + p = psutil.Process(self.pid) + # Note: wmi reports the exe as a lower case string. + # Being Windows paths case-insensitive we ignore that. + assert p.exe().lower() == w.ExecutablePath.lower() + + def test_cmdline(self): + w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + p = psutil.Process(self.pid) + assert ' '.join(p.cmdline()) == w.CommandLine.replace('"', '') + + def test_username(self): + w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + p = psutil.Process(self.pid) + domain, _, username = w.GetOwner() + username = f"{domain}\\{username}" + assert p.username() == username + + @retry_on_failure() + def test_memory_rss(self): + w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + p = psutil.Process(self.pid) + rss = p.memory_info().rss + assert rss == int(w.WorkingSetSize) + + @retry_on_failure() + def test_memory_vms(self): + w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + p = psutil.Process(self.pid) + vms = p.memory_info().vms + # http://msdn.microsoft.com/en-us/library/aa394372(VS.85).aspx + # ...claims that PageFileUsage is represented in Kilo + # bytes but funnily enough on certain platforms bytes are + # returned instead. + wmi_usage = int(w.PageFileUsage) + if vms not in {wmi_usage, wmi_usage * 1024}: + raise self.fail(f"wmi={wmi_usage}, psutil={vms}") + + def test_create_time(self): + w = wmi.WMI().Win32_Process(ProcessId=self.pid)[0] + p = psutil.Process(self.pid) + wmic_create = str(w.CreationDate.split('.')[0]) + psutil_create = time.strftime( + "%Y%m%d%H%M%S", time.localtime(p.create_time()) + ) + assert wmic_create == psutil_create + + +# --- + + +@pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") +class TestDualProcessImplementation(PsutilTestCase): + """Certain APIs on Windows have 2 internal implementations, one + based on documented Windows APIs, another one based + NtQuerySystemInformation() which gets called as fallback in + case the first fails because of limited permission error. + Here we test that the two methods return the exact same value, + see: + https://github.com/giampaolo/psutil/issues/304. + """ + + @classmethod + def setUpClass(cls): + cls.pid = spawn_testproc().pid + + @classmethod + def tearDownClass(cls): + terminate(cls.pid) + + def test_memory_info(self): + mem_1 = psutil.Process(self.pid).memory_info() + with mock.patch( + "psutil._psplatform.cext.proc_memory_info", + side_effect=PermissionError, + ) as fun: + mem_2 = psutil.Process(self.pid).memory_info() + assert len(mem_1) == len(mem_2) + for i in range(len(mem_1)): + assert mem_1[i] >= 0 + assert mem_2[i] >= 0 + assert abs(mem_1[i] - mem_2[i]) < 512 + assert fun.called + + def test_create_time(self): + ctime = psutil.Process(self.pid).create_time() + with mock.patch( + "psutil._psplatform.cext.proc_times", + side_effect=PermissionError, + ) as fun: + assert psutil.Process(self.pid).create_time() == ctime + assert fun.called + + def test_cpu_times(self): + cpu_times_1 = psutil.Process(self.pid).cpu_times() + with mock.patch( + "psutil._psplatform.cext.proc_times", + side_effect=PermissionError, + ) as fun: + cpu_times_2 = psutil.Process(self.pid).cpu_times() + assert fun.called + assert abs(cpu_times_1.user - cpu_times_2.user) < 0.01 + assert abs(cpu_times_1.system - cpu_times_2.system) < 0.01 + + def test_io_counters(self): + io_counters_1 = psutil.Process(self.pid).io_counters() + with mock.patch( + "psutil._psplatform.cext.proc_io_counters", + side_effect=PermissionError, + ) as fun: + io_counters_2 = psutil.Process(self.pid).io_counters() + for i in range(len(io_counters_1)): + assert abs(io_counters_1[i] - io_counters_2[i]) < 5 + assert fun.called + + def test_num_handles(self): + num_handles = psutil.Process(self.pid).num_handles() + with mock.patch( + "psutil._psplatform.cext.proc_num_handles", + side_effect=PermissionError, + ) as fun: + assert psutil.Process(self.pid).num_handles() == num_handles + assert fun.called + + def test_cmdline(self): + for pid in psutil.pids(): + try: + a = cext.proc_cmdline(pid, use_peb=True) + b = cext.proc_cmdline(pid, use_peb=False) + except OSError as err: + err = convert_oserror(err) + if not isinstance( + err, (psutil.AccessDenied, psutil.NoSuchProcess) + ): + raise + else: + assert a == b + + +@pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") +class RemoteProcessTestCase(PsutilTestCase): + """Certain functions require calling ReadProcessMemory. + This trivially works when called on the current process. + Check that this works on other processes, especially when they + have a different bitness. + """ + + @staticmethod + def find_other_interpreter(): + # find a python interpreter that is of the opposite bitness from us + code = "import sys; sys.stdout.write(str(sys.maxsize > 2**32))" + + # XXX: a different and probably more stable approach might be to access + # the registry but accessing 64 bit paths from a 32 bit process + for filename in glob.glob(r"C:\Python*\python.exe"): + proc = subprocess.Popen( + args=[filename, "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + output, _ = proc.communicate() + proc.wait() + if output == str(not IS_64BIT): + return filename + + test_args = ["-c", "import sys; sys.stdin.read()"] + + def setUp(self): + super().setUp() + + other_python = self.find_other_interpreter() + if other_python is None: + raise pytest.skip( + "could not find interpreter with opposite bitness" + ) + if IS_64BIT: + self.python64 = sys.executable + self.python32 = other_python + else: + self.python64 = other_python + self.python32 = sys.executable + + env = os.environ.copy() + env["THINK_OF_A_NUMBER"] = str(os.getpid()) + self.proc32 = self.spawn_testproc( + [self.python32] + self.test_args, env=env, stdin=subprocess.PIPE + ) + self.proc64 = self.spawn_testproc( + [self.python64] + self.test_args, env=env, stdin=subprocess.PIPE + ) + + def tearDown(self): + super().tearDown() + self.proc32.communicate() + self.proc64.communicate() + + def test_cmdline_32(self): + p = psutil.Process(self.proc32.pid) + assert len(p.cmdline()) == 3 + assert p.cmdline()[1:] == self.test_args + + def test_cmdline_64(self): + p = psutil.Process(self.proc64.pid) + assert len(p.cmdline()) == 3 + assert p.cmdline()[1:] == self.test_args + + def test_cwd_32(self): + p = psutil.Process(self.proc32.pid) + assert p.cwd() == os.getcwd() + + def test_cwd_64(self): + p = psutil.Process(self.proc64.pid) + assert p.cwd() == os.getcwd() + + def test_environ_32(self): + p = psutil.Process(self.proc32.pid) + e = p.environ() + assert "THINK_OF_A_NUMBER" in e + assert e["THINK_OF_A_NUMBER"] == str(os.getpid()) + + def test_environ_64(self): + p = psutil.Process(self.proc64.pid) + try: + p.environ() + except psutil.AccessDenied: + pass + + +# =================================================================== +# Windows services +# =================================================================== + + +@pytest.mark.skipif(not WINDOWS, reason="WINDOWS only") +class TestServices(PsutilTestCase): + def test_win_service_iter(self): + valid_statuses = { + "running", + "paused", + "start", + "pause", + "continue", + "stop", + "stopped", + } + valid_start_types = {"automatic", "manual", "disabled"} + valid_statuses = { + "running", + "paused", + "start_pending", + "pause_pending", + "continue_pending", + "stop_pending", + "stopped", + } + for serv in psutil.win_service_iter(): + data = serv.as_dict() + assert isinstance(data['name'], str) + assert data['name'].strip() + assert isinstance(data['display_name'], str) + assert isinstance(data['username'], str) + assert data['status'] in valid_statuses + if data['pid'] is not None: + psutil.Process(data['pid']) + assert isinstance(data['binpath'], str) + assert isinstance(data['username'], str) + assert isinstance(data['start_type'], str) + assert data['start_type'] in valid_start_types + assert data['status'] in valid_statuses + assert isinstance(data['description'], str) + pid = serv.pid() + if pid is not None: + p = psutil.Process(pid) + assert p.is_running() + # win_service_get + s = psutil.win_service_get(serv.name()) + # test __eq__ + assert serv == s + + def test_win_service_get(self): + ERROR_SERVICE_DOES_NOT_EXIST = ( + psutil._psplatform.cext.ERROR_SERVICE_DOES_NOT_EXIST + ) + ERROR_ACCESS_DENIED = psutil._psplatform.cext.ERROR_ACCESS_DENIED + + name = next(psutil.win_service_iter()).name() + with pytest.raises(psutil.NoSuchProcess) as cm: + psutil.win_service_get(name + '???') + assert cm.value.name == name + '???' + + # test NoSuchProcess + service = psutil.win_service_get(name) + exc = OSError(0, "msg", 0) + exc.winerror = ERROR_SERVICE_DOES_NOT_EXIST + with mock.patch( + "psutil._psplatform.cext.winservice_query_status", side_effect=exc + ): + with pytest.raises(psutil.NoSuchProcess): + service.status() + with mock.patch( + "psutil._psplatform.cext.winservice_query_config", side_effect=exc + ): + with pytest.raises(psutil.NoSuchProcess): + service.username() + + # test AccessDenied + exc = OSError(0, "msg", 0) + exc.winerror = ERROR_ACCESS_DENIED + with mock.patch( + "psutil._psplatform.cext.winservice_query_status", side_effect=exc + ): + with pytest.raises(psutil.AccessDenied): + service.status() + with mock.patch( + "psutil._psplatform.cext.winservice_query_config", side_effect=exc + ): + with pytest.raises(psutil.AccessDenied): + service.username() + + # test __str__ and __repr__ + assert service.name() in str(service) + assert service.display_name() in str(service) + assert service.name() in repr(service) + assert service.display_name() in repr(service) diff --git a/lib/python3.12/site-packages/sentencepiece/__init__.py b/lib/python3.12/site-packages/sentencepiece/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..02379dd36dad021a82d4b3a85ad1598f01184758 --- /dev/null +++ b/lib/python3.12/site-packages/sentencepiece/__init__.py @@ -0,0 +1,1230 @@ +# This file was automatically generated by SWIG (https://www.swig.org). +# Version 4.3.0 +# +# Do not make changes to this file unless you know what you are doing - modify +# the SWIG interface file instead. + +from sys import version_info as _swig_python_version_info +# Import the low-level C/C++ module +if __package__ or "." in __name__: + from . import _sentencepiece +else: + import _sentencepiece + +try: + import builtins as __builtin__ +except ImportError: + import __builtin__ + +def _swig_repr(self): + try: + strthis = "proxy of " + self.this.__repr__() + except __builtin__.Exception: + strthis = "" + return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,) + + +def _swig_setattr_nondynamic_instance_variable(set): + def set_instance_attr(self, name, value): + if name == "this": + set(self, name, value) + elif name == "thisown": + self.this.own(value) + elif hasattr(self, name) and isinstance(getattr(type(self), name), property): + set(self, name, value) + else: + raise AttributeError("You cannot add instance attributes to %s" % self) + return set_instance_attr + + +def _swig_setattr_nondynamic_class_variable(set): + def set_class_attr(cls, name, value): + if hasattr(cls, name) and not isinstance(getattr(cls, name), property): + set(cls, name, value) + else: + raise AttributeError("You cannot add class attributes to %s" % cls) + return set_class_attr + + +def _swig_add_metaclass(metaclass): + """Class decorator for adding a metaclass to a SWIG wrapped class - a slimmed down version of six.add_metaclass""" + def wrapper(cls): + return metaclass(cls.__name__, cls.__bases__, cls.__dict__.copy()) + return wrapper + + +class _SwigNonDynamicMeta(type): + """Meta class to enforce nondynamic attributes (no new attributes) for a class""" + __setattr__ = _swig_setattr_nondynamic_class_variable(type.__setattr__) + + +class ImmutableSentencePieceText_ImmutableSentencePiece(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_swiginit(self, _sentencepiece.new_ImmutableSentencePieceText_ImmutableSentencePiece()) + __swig_destroy__ = _sentencepiece.delete_ImmutableSentencePieceText_ImmutableSentencePiece + + def _piece(self): + return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__piece(self) + + def _surface(self): + return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__surface(self) + + def _id(self): + return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__id(self) + + def _begin(self): + return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__begin(self) + + def _end(self): + return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__end(self) + + def _surface_as_bytes(self): + return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes(self) + + def _piece_as_bytes(self): + return _sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes(self) + + piece = property(_piece) + piece_as_bytes = property(_piece_as_bytes) + surface = property(_surface) + surface_as_bytes = property(_surface_as_bytes) + id = property(_id) + begin = property(_begin) + end = property(_end) + + def __str__(self): + return ('piece: \"{}\"\n' + 'id: {}\n' + 'surface: \"{}\"\n' + 'begin: {}\n' + 'end: {}\n').format(self.piece, self.id, self.surface, + self.begin, self.end) + + def __eq__(self, other): + return self.piece == other.piece and self.id == other.id and self.surface == other.surface and self.begin == other.begin and self.end == other.end + + def __hash__(self): + return hash(str(self)) + + __repr__ = __str__ + + +# Register ImmutableSentencePieceText_ImmutableSentencePiece in _sentencepiece: +_sentencepiece.ImmutableSentencePieceText_ImmutableSentencePiece_swigregister(ImmutableSentencePieceText_ImmutableSentencePiece) +class ImmutableSentencePieceText(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _sentencepiece.ImmutableSentencePieceText_swiginit(self, _sentencepiece.new_ImmutableSentencePieceText()) + __swig_destroy__ = _sentencepiece.delete_ImmutableSentencePieceText + + def _pieces_size(self): + return _sentencepiece.ImmutableSentencePieceText__pieces_size(self) + + def _pieces(self, index): + return _sentencepiece.ImmutableSentencePieceText__pieces(self, index) + + def _text(self): + return _sentencepiece.ImmutableSentencePieceText__text(self) + + def _score(self): + return _sentencepiece.ImmutableSentencePieceText__score(self) + + def SerializeAsString(self): + return _sentencepiece.ImmutableSentencePieceText_SerializeAsString(self) + + def _text_as_bytes(self): + return _sentencepiece.ImmutableSentencePieceText__text_as_bytes(self) + + text = property(_text) + text_as_bytes = property(_text_as_bytes) + score = property(_score) + + class ImmutableSentencePieceIterator: + def __init__(self, proto): + self.proto = proto + self.len = self.proto._pieces_size() + + def __len__(self): + return self.len + + def __getitem__(self, index): + if isinstance(index, slice): + return [self.proto._pieces(i) for i in range(self.len)][index.start:index.stop:index.step] + if index < 0: + index = index + self.len + if index < 0 or index >= self.len: + raise IndexError('piece index is out of range') + return self.proto._pieces(index) + + def __str__(self): + return '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self]) + + __repr__ = __str__ + + @property + def pieces(self): + return ImmutableSentencePieceText.ImmutableSentencePieceIterator(self) + + def __eq__(self, other): + return self.SerializeAsString() == other.SerializeAsString() + + def __hash__(self): + return hash(self.SerializeAsString()) + + def __str__(self): + return ('text: \"{}\"\n' + 'score: {}\n' + '{}').format(self.text, self.score, + '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self.pieces])) + + __repr__ = __str__ + + +# Register ImmutableSentencePieceText in _sentencepiece: +_sentencepiece.ImmutableSentencePieceText_swigregister(ImmutableSentencePieceText) +class ImmutableNBestSentencePieceText(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _sentencepiece.ImmutableNBestSentencePieceText_swiginit(self, _sentencepiece.new_ImmutableNBestSentencePieceText()) + __swig_destroy__ = _sentencepiece.delete_ImmutableNBestSentencePieceText + + def _nbests_size(self): + return _sentencepiece.ImmutableNBestSentencePieceText__nbests_size(self) + + def _nbests(self, index): + return _sentencepiece.ImmutableNBestSentencePieceText__nbests(self, index) + + def SerializeAsString(self): + return _sentencepiece.ImmutableNBestSentencePieceText_SerializeAsString(self) + + class ImmutableSentencePieceTextIterator: + def __init__(self, proto): + self.proto = proto + self.len = self.proto._nbests_size() + + def __len__(self): + return self.len + + def __getitem__(self, index): + if isinstance(index, slice): + return [self.proto._nbests(i) for i in range(self.len)][index.start:index.stop:index.step] + if index < 0: + index = index + self.len + if index < 0 or index >= self.len: + raise IndexError('nbests index is out of range') + return self.proto._nbests(index) + + def __str__(self): + return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self]) + + __repr__ = __str__ + + @property + def nbests(self): + return ImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator(self) + + def __eq__(self, other): + return self.SerializeAsString() == other.SerializeAsString() + + def __hash__(self): + return hash(self.SerializeAsString()) + + def __str__(self): + return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self.nbests]) + + __repr__ = __str__ + + +# Register ImmutableNBestSentencePieceText in _sentencepiece: +_sentencepiece.ImmutableNBestSentencePieceText_swigregister(ImmutableNBestSentencePieceText) +class SentencePieceProcessor(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _sentencepiece.SentencePieceProcessor_swiginit(self, _sentencepiece.new_SentencePieceProcessor()) + __swig_destroy__ = _sentencepiece.delete_SentencePieceProcessor + + def LoadFromSerializedProto(self, serialized): + return _sentencepiece.SentencePieceProcessor_LoadFromSerializedProto(self, serialized) + + def SetEncodeExtraOptions(self, extra_option): + return _sentencepiece.SentencePieceProcessor_SetEncodeExtraOptions(self, extra_option) + + def SetDecodeExtraOptions(self, extra_option): + return _sentencepiece.SentencePieceProcessor_SetDecodeExtraOptions(self, extra_option) + + def SetVocabulary(self, valid_vocab): + return _sentencepiece.SentencePieceProcessor_SetVocabulary(self, valid_vocab) + + def ResetVocabulary(self): + return _sentencepiece.SentencePieceProcessor_ResetVocabulary(self) + + def LoadVocabulary(self, filename, threshold): + return _sentencepiece.SentencePieceProcessor_LoadVocabulary(self, filename, threshold) + + def CalculateEntropy(self, *args): + return _sentencepiece.SentencePieceProcessor_CalculateEntropy(self, *args) + + def GetPieceSize(self): + return _sentencepiece.SentencePieceProcessor_GetPieceSize(self) + + def PieceToId(self, piece): + return _sentencepiece.SentencePieceProcessor_PieceToId(self, piece) + + def IdToPiece(self, id): + return _sentencepiece.SentencePieceProcessor_IdToPiece(self, id) + + def GetScore(self, id): + return _sentencepiece.SentencePieceProcessor_GetScore(self, id) + + def IsUnknown(self, id): + return _sentencepiece.SentencePieceProcessor_IsUnknown(self, id) + + def IsControl(self, id): + return _sentencepiece.SentencePieceProcessor_IsControl(self, id) + + def IsUnused(self, id): + return _sentencepiece.SentencePieceProcessor_IsUnused(self, id) + + def IsByte(self, id): + return _sentencepiece.SentencePieceProcessor_IsByte(self, id) + + def unk_id(self): + return _sentencepiece.SentencePieceProcessor_unk_id(self) + + def bos_id(self): + return _sentencepiece.SentencePieceProcessor_bos_id(self) + + def eos_id(self): + return _sentencepiece.SentencePieceProcessor_eos_id(self) + + def pad_id(self): + return _sentencepiece.SentencePieceProcessor_pad_id(self) + + def serialized_model_proto(self): + return _sentencepiece.SentencePieceProcessor_serialized_model_proto(self) + + def LoadFromFile(self, arg): + return _sentencepiece.SentencePieceProcessor_LoadFromFile(self, arg) + + def _EncodeAsIds(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsIds(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _EncodeAsPieces(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsPieces(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _EncodeAsSerializedProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsSerializedProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _EncodeAsImmutableProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsImmutableProto(self, text, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _EncodeAsIdsBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsIdsBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _EncodeAsPiecesBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsPiecesBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _EncodeAsSerializedProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsSerializedProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _EncodeAsImmutableProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__EncodeAsImmutableProtoBatch(self, ins, num_threads, enable_sampling, nbest_size, alpha, add_bos, add_eos, reverse, emit_unk_piece) + + def _DecodeIds(self, ids): + return _sentencepiece.SentencePieceProcessor__DecodeIds(self, ids) + + def _DecodeIdsAsBytes(self, ids): + return _sentencepiece.SentencePieceProcessor__DecodeIdsAsBytes(self, ids) + + def _DecodePieces(self, pieces): + return _sentencepiece.SentencePieceProcessor__DecodePieces(self, pieces) + + def _DecodeIdsAsSerializedProto(self, ids): + return _sentencepiece.SentencePieceProcessor__DecodeIdsAsSerializedProto(self, ids) + + def _DecodePiecesAsSerializedProto(self, pieces): + return _sentencepiece.SentencePieceProcessor__DecodePiecesAsSerializedProto(self, pieces) + + def _DecodeIdsAsImmutableProto(self, ids): + return _sentencepiece.SentencePieceProcessor__DecodeIdsAsImmutableProto(self, ids) + + def _DecodePiecesAsImmutableProto(self, pieces): + return _sentencepiece.SentencePieceProcessor__DecodePiecesAsImmutableProto(self, pieces) + + def _DecodeIdsBatch(self, ins, num_threads): + return _sentencepiece.SentencePieceProcessor__DecodeIdsBatch(self, ins, num_threads) + + def _DecodeIdsAsBytesBatch(self, ins, num_threads): + return _sentencepiece.SentencePieceProcessor__DecodeIdsAsBytesBatch(self, ins, num_threads) + + def _DecodeIdsAsSerializedProtoBatch(self, ins, num_threads): + return _sentencepiece.SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch(self, ins, num_threads) + + def _DecodeIdsAsImmutableProtoBatch(self, ins, num_threads): + return _sentencepiece.SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch(self, ins, num_threads) + + def _DecodePiecesBatch(self, ins, num_threads): + return _sentencepiece.SentencePieceProcessor__DecodePiecesBatch(self, ins, num_threads) + + def _DecodePiecesAsSerializedProtoBatch(self, ins, num_threads): + return _sentencepiece.SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch(self, ins, num_threads) + + def _DecodePiecesAsImmutableProtoBatch(self, ins, num_threads): + return _sentencepiece.SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch(self, ins, num_threads) + + def _NBestEncodeAsIds(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__NBestEncodeAsIds(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece) + + def _NBestEncodeAsPieces(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__NBestEncodeAsPieces(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece) + + def _NBestEncodeAsSerializedProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__NBestEncodeAsSerializedProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece) + + def _NBestEncodeAsImmutableProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__NBestEncodeAsImmutableProto(self, text, nbest_size, add_bos, add_eos, reverse, emit_unk_piece) + + def _SampleEncodeAndScoreAsIds(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsIds(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece) + + def _SampleEncodeAndScoreAsPieces(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsPieces(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece) + + def _SampleEncodeAndScoreAsSerializedProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece) + + def _SampleEncodeAndScoreAsImmutableProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece): + return _sentencepiece.SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto(self, text, num_samples, alpha, wor, include_best, add_bos, add_eos, reverse, emit_unk_piece) + + def _Normalize(self, text): + return _sentencepiece.SentencePieceProcessor__Normalize(self, text) + + def _NormalizeWithOffsets(self, text): + return _sentencepiece.SentencePieceProcessor__NormalizeWithOffsets(self, text) + + def _CalculateEntropy(self, text, alpha): + return _sentencepiece.SentencePieceProcessor__CalculateEntropy(self, text, alpha) + + def _CalculateEntropyBatch(self, ins, alpha, num_threads): + return _sentencepiece.SentencePieceProcessor__CalculateEntropyBatch(self, ins, alpha, num_threads) + + def _OverrideNormalizerSpec(self, args): + return _sentencepiece.SentencePieceProcessor__OverrideNormalizerSpec(self, args) + + def Init(self, + model_file=None, + model_proto=None, + out_type=int, + add_bos=False, + add_eos=False, + reverse=False, + emit_unk_piece=False, + enable_sampling=False, + nbest_size=-1, + alpha=0.1, + num_threads=-1): + """Initialzie sentencepieceProcessor. + + Args: + model_file: The sentencepiece model file path. + model_proto: The sentencepiece model serialized proto. + out_type: output type. int or str. + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after + reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. + nbest_size = {0,1}: No sampling is performed. + nbest_size > 1: samples from the nbest_size results. + nbest_size < 0: assuming that nbest_size is infinite and samples + from the all hypothesis (lattice) using + forward-filtering-and-backward-sampling algorithm. + alpha: Soothing parameter for unigram sampling, and dropout probability of + merge operations for BPE-dropout. + num_threads: number of threads in batch processing (Default = -1, auto-detected) + """ + + _sentencepiece_processor_init_native(self) + self._out_type = out_type + self._add_bos = add_bos + self._add_eos = add_eos + self._reverse = reverse + self._emit_unk_piece = emit_unk_piece + self._enable_sampling = enable_sampling + self._nbest_size = nbest_size + self._alpha = alpha + self._num_threads = num_threads + if model_file or model_proto: + self.Load(model_file=model_file, model_proto=model_proto) + + + def Encode(self, + input, + out_type=None, + add_bos=None, + add_eos=None, + reverse=None, + emit_unk_piece=None, + enable_sampling=None, + nbest_size=None, + alpha=None, + num_threads=None): + """Encode text input to segmented ids or tokens. + + Args: + input: input string. accepsts list of string. + out_type: output type. int or str. + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after + reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. + nbest_size = {0,1}: No sampling is performed. + nbest_size > 1: samples from the nbest_size results. + nbest_size < 0: assuming that nbest_size is infinite and samples + from the all hypothesis (lattice) using + forward-filtering-and-backward-sampling algorithm. + alpha: Soothing parameter for unigram sampling, and merge probability for + BPE-dropout (probablity 'p' in BPE-dropout paper). + num_threads: the number of threads used in the batch processing (Default = -1). + """ + + if out_type is None: + out_type = self._out_type + if add_bos is None: + add_bos = self._add_bos + if add_eos is None: + add_eos = self._add_eos + if reverse is None: + reverse = self._reverse + if emit_unk_piece is None: + emit_unk_piece = self._emit_unk_piece + if enable_sampling is None: + enable_sampling = self._enable_sampling + if nbest_size is None: + nbest_size = self._nbest_size + if alpha is None: + alpha = self._alpha + if num_threads is None: + num_threads = self._num_threads + + if enable_sampling == True and (nbest_size is None or nbest_size == 0 or + nbest_size == 1 or alpha is None): + raise RuntimeError( + 'When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", ' + 'and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. ' + 'when "nbest_size = -1" , this method samples from all candidates on the lattice ' + 'instead of nbest segmentations.' + ) + + if num_threads is None or type(num_threads) is not int: + raise RuntimeError('num_threads must be int') + + if type(input) is list: + if out_type is int: + return self._EncodeAsIdsBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._EncodeAsPiecesBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'serialized_proto' or out_type == 'proto': + return self._EncodeAsSerializedProtoBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'immutable_proto': + return self._EncodeAsImmutableProtoBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + + if out_type is int: + return self._EncodeAsIds(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._EncodeAsPieces(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'serialized_proto' or out_type == 'proto': + return self._EncodeAsSerializedProto(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'immutable_proto': + return self._EncodeAsImmutableProto(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + + raise RuntimeError('unknown out_type={}'.format(out_type)) + return None + + + def EncodeAsPieces(self, input, **kwargs): + return self.Encode(input=input, out_type=str, **kwargs) + + + def EncodeAsIds(self, input, **kwargs): + return self.Encode(input=input, out_type=int, **kwargs) + + + def EncodeAsSerializedProto(self, input, **kwargs): + return self.Encode(input=input, out_type='serialized_proto', **kwargs) + + + def EncodeAsImmutableProto(self, input, **kwargs): + return self.Encode(input=input, out_type='immutable_proto', **kwargs) + + + def SampleEncodeAsPieces(self, input, nbest_size=None, alpha=None, **kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type=str, enable_sampling=True, **kwargs) + + + def SampleEncodeAsIds(self, input, nbest_size=None, alpha=None,**kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type=int, enable_sampling=True, **kwargs) + + + def SampleEncodeAsSerializedProto(self, input, nbest_size=None, alpha=None, **kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type='serialized_proto', enable_sampling=True, **kwargs) + + + def SampleEncodeAsImmutableProto(self, input, nbest_size=None, alpha=None, **kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type='immutable_proto', enable_sampling=True, **kwargs) + + + def NBestEncode(self, + input, + out_type=None, + add_bos=None, + add_eos=None, + reverse=None, + emit_unk_piece=None, + nbest_size=None): + """NBestEncode text input to segmented ids or tokens. + + Args: + input: input string. accepsts list of string. + out_type: output type. int or str. + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + nbest_size: nbest size + """ + + if out_type is None: + out_type = self._out_type + if add_bos is None: + add_bos = self._add_bos + if add_eos is None: + add_eos = self._add_eos + if reverse is None: + reverse = self._reverse + if emit_unk_piece is None: + emit_unk_piece = self._emit_unk_piece + if nbest_size is None: + nbest_size = self._nbest_size + + if nbest_size <= 0: + nbest_size=1 + + def _encode(text): + if out_type is int: + return self._NBestEncodeAsIds(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._NBestEncodeAsPieces(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'serialized_proto' or out_type == 'proto': + return self._NBestEncodeAsSerializedProto(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'immutable_proto': + return self._NBestEncodeAsImmutableProto(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + + raise RuntimeError('unknown out_type') + + if type(input) is list: + return [_encode(n) for n in input] + + return _encode(input) + + + def NBestEncodeAsPieces(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type=str, **kwargs) + + + def NBestEncodeAsIds(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type=int, **kwargs) + + + def NBestEncodeAsSerializedProto(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type='serialized_proto', **kwargs) + + + def NBestEncodeAsImmutableProto(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type='immutable_proto', **kwargs) + + + def SampleEncodeAndScore(self, + input, + out_type=None, + add_bos=None, + add_eos=None, + reverse=None, + emit_unk_piece=None, + num_samples=None, + alpha=None, + wor=None, + include_best=None): + """SampleEncodeAndScore text input to segmented ids or tokens. + + Args: + input: input string. accepsts list of string. + out_type: output type. int or str or 'serialized_proto' or 'immutable_proto' + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + num_samples: How many samples to return (Default = 1) + alpha: inverse temperature for sampling + wor: whether to sample without replacement (Default = false) + include_best: whether to include the best tokenization, requires wor=True (Default = false) + """ + + if out_type is None: + out_type = self._out_type + if add_bos is None: + add_bos = self._add_bos + if add_eos is None: + add_eos = self._add_eos + if reverse is None: + reverse = self._reverse + if emit_unk_piece is None: + emit_unk_piece = self._emit_unk_piece + if num_samples is None: + num_samples = 1 + if alpha is None: + alpha = 1. + if wor is None: + wor = False + if include_best is None: + include_best = False + + if num_samples <= 0: + raise RuntimeError('num_examples must be positive') + + if include_best and not wor: + raise RuntimeError('When include_best is True, We must specify "wor = True".') + + + def _encode(text): + if out_type is int: + return self._SampleEncodeAndScoreAsIds(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._SampleEncodeAndScoreAsPieces(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + + if out_type == 'serialized_proto' or out_type == 'proto': + return self._SampleEncodeAndScoreAsSerializedProto(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + + if out_type == 'immutable_proto': + return self._SampleEncodeAndScoreAsImmutableProto(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + + raise RuntimeError('unknown output type') + + + if type(input) is list: + return [_encode(n) for n in input] + + return _encode(input) + + + def SampleEncodeAndScoreAsPieces(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type=str, **kwargs) + + + def SampleEncodeAndScoreAsIds(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type=int, **kwargs) + + + def SampleEncodeAndScoreAsSerializedProto(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type='serialized_proto', **kwargs) + + + def SampleEncodeAndScoreAsImmutableProto(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type='immutable_proto', **kwargs) + + + def Decode(self, input, out_type=str, num_threads=None): + """Decode processed id or token sequences. + + Args: + out_type: output type. str, bytes or 'serialized_proto' or 'immutable_proto' (Default = str) + num_threads: the number of threads used in the batch processing (Default = -1). + """ + + if num_threads is None: + num_threads = self._num_threads + + if num_threads is None or type(num_threads) is not int: + raise RuntimeError('num_threads must be int') + + if not input: + return '' + + if out_type is str: + if type(input) is int: + return self._DecodeIds([input]) + if type(input) is str: + return self._DecodePieces([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIds(input) + if type(input[0]) is str: + return self._DecodePieces(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesBatch(input, num_threads) + + if out_type is bytes: + if type(input) is int: + return self._DecodeIdsAsBytes([input]) + if type(input) is str: + return self._DecodePieces([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIdsAsBytes(input) + if type(input[0]) is str: + return self._DecodePieces(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsAsBytesBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesBatch(input, num_threads) + + if out_type == 'serialized_proto': + if type(input) is int: + return self._DecodeIdsAsSerializedProto([input]) + if type(input) is str: + return self._DecodePiecesAsSerializedProto([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIdsAsSerializedProto(input) + if type(input[0]) is str: + return self._DecodePiecesAsSerializedProto(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsAsSerializedProtoBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesAsSerializedProtoBatch(input, num_threads) + + + if out_type == 'immutable_proto': + if type(input) is int: + return self._DecodeIdsAsImmutableProto([input]) + if type(input) is str: + return self._DecodePiecesAsImmutableProto([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIdsAsImmutableProto(input) + if type(input[0]) is str: + return self._DecodePiecesAsImmutableProto(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsAsImmutableProtoBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesAsImmutableProtoBatch(input, num_threads) + + + raise RuntimeError('unknown output or input type') + return None + + + def DecodePieces(self, input, out_type=str, **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodeIds(self, input, out_type=str, **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodePiecesAsSerializedProto(self, input, out_type='serialized_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodeIdsAsSerializedProto(self, input, out_type='serialized_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodePiecesAsImmutableProto(self, input, out_type='immutable_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodeIdsAsImmutableProto(self, input, out_type='immutable_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def CalculateEntropy(self, input, alpha, num_threads=None): + """Calculate sentence entropy""" + if type(input) is list: + if num_threads is None: + num_threads = self._num_threads + if num_threads is None or type(num_threads) is not int: + raise RuntimeError('num_threads must be int') + return self._CalculateEntropyBatch(input, alpha, num_threads) + + return self._CalculateEntropy(input, alpha) + + + def Normalize(self, input, with_offsets=None): + def _normalize(text): + if with_offsets: + return self._NormalizeWithOffsets(text) + return self._Normalize(text) + + if type(input) is list: + return [_normalize(x) for x in input] + return _normalize(input) + + def OverrideNormalizerSpec(self, **kwargs): + new_kwargs = {} + for key, value in kwargs.items(): + new_kwargs[key] = str(value) + return self._OverrideNormalizerSpec(new_kwargs) + + + def piece_size(self): + return self.GetPieceSize() + + + def vocab_size(self): + return self.GetPieceSize() + + + def __getstate__(self): + return self.serialized_model_proto() + + + def __setstate__(self, serialized_model_proto): + self.__init__() + self.LoadFromSerializedProto(serialized_model_proto) + + + def __len__(self): + return self.GetPieceSize() + + + def __getitem__(self, piece): + return self.PieceToId(piece) + + + def Load(self, model_file=None, model_proto=None): + """Overwride SentencePieceProcessor.Load to support both model_file and model_proto. + + Args: + model_file: The sentencepiece model file path. + model_proto: The sentencepiece model serialized proto. Either `model_file` + or `model_proto` must be set. + """ + if model_file and model_proto: + raise RuntimeError('model_file and model_proto must be exclusive.') + if model_proto: + return self.LoadFromSerializedProto(model_proto) + return self.LoadFromFile(model_file) + + +# Register SentencePieceProcessor in _sentencepiece: +_sentencepiece.SentencePieceProcessor_swigregister(SentencePieceProcessor) + +def SetRandomGeneratorSeed(seed): + return _sentencepiece.SetRandomGeneratorSeed(seed) + +def SetMinLogLevel(v): + return _sentencepiece.SetMinLogLevel(v) +class SentencePieceTrainer(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + + def __init__(self, *args, **kwargs): + raise AttributeError("No constructor defined") + __repr__ = _swig_repr + + @staticmethod + def _TrainFromString(arg): + return _sentencepiece.SentencePieceTrainer__TrainFromString(arg) + + @staticmethod + def _TrainFromMap(args): + return _sentencepiece.SentencePieceTrainer__TrainFromMap(args) + + @staticmethod + def _TrainFromMap2(args, iter): + return _sentencepiece.SentencePieceTrainer__TrainFromMap2(args, iter) + + @staticmethod + def _TrainFromMap3(args): + return _sentencepiece.SentencePieceTrainer__TrainFromMap3(args) + + @staticmethod + def _TrainFromMap4(args, iter): + return _sentencepiece.SentencePieceTrainer__TrainFromMap4(args, iter) + + @staticmethod + def _Train(arg=None, **kwargs): + """Train Sentencepiece model. Accept both kwargs and legacy string arg.""" + if arg is not None and type(arg) is str: + return SentencePieceTrainer._TrainFromString(arg) + + def _encode(value): + """Encode value to CSV..""" + if type(value) is list: + if sys.version_info[0] == 3: + f = StringIO() + else: + f = BytesIO() + writer = csv.writer(f, lineterminator='') + writer.writerow([str(v) for v in value]) + return f.getvalue() + else: + return str(value) + + sentence_iterator = None + model_writer = None + new_kwargs = {} + for key, value in kwargs.items(): + if key in ['sentence_iterator', 'sentence_reader']: + sentence_iterator = value + elif key in ['model_writer']: + model_writer = value + else: + new_kwargs[key] = _encode(value) + + if model_writer: + if sentence_iterator: + model_proto = SentencePieceTrainer._TrainFromMap4(new_kwargs, + sentence_iterator) + else: + model_proto = SentencePieceTrainer._TrainFromMap3(new_kwargs) + model_writer.write(model_proto) + else: + if sentence_iterator: + return SentencePieceTrainer._TrainFromMap2(new_kwargs, sentence_iterator) + else: + return SentencePieceTrainer._TrainFromMap(new_kwargs) + + return None + + @staticmethod + def Train(arg=None, logstream=None, **kwargs): + with _LogStream(ostream=logstream): + SentencePieceTrainer._Train(arg=arg, **kwargs) + + +# Register SentencePieceTrainer in _sentencepiece: +_sentencepiece.SentencePieceTrainer_swigregister(SentencePieceTrainer) +class SentencePieceNormalizer(object): + thisown = property(lambda x: x.this.own(), lambda x, v: x.this.own(v), doc="The membership flag") + __repr__ = _swig_repr + + def __init__(self): + _sentencepiece.SentencePieceNormalizer_swiginit(self, _sentencepiece.new_SentencePieceNormalizer()) + __swig_destroy__ = _sentencepiece.delete_SentencePieceNormalizer + + def LoadFromSerializedProto(self, serialized): + return _sentencepiece.SentencePieceNormalizer_LoadFromSerializedProto(self, serialized) + + def LoadFromRuleTSV(self, filename): + return _sentencepiece.SentencePieceNormalizer_LoadFromRuleTSV(self, filename) + + def LoadFromRuleName(self, name): + return _sentencepiece.SentencePieceNormalizer_LoadFromRuleName(self, name) + + def serialized_model_proto(self): + return _sentencepiece.SentencePieceNormalizer_serialized_model_proto(self) + + def LoadFromFile(self, arg): + return _sentencepiece.SentencePieceNormalizer_LoadFromFile(self, arg) + + def _Normalize(self, text): + return _sentencepiece.SentencePieceNormalizer__Normalize(self, text) + + def _NormalizeWithOffsets(self, text): + return _sentencepiece.SentencePieceNormalizer__NormalizeWithOffsets(self, text) + + def _SetProtoField(self, name, value): + return _sentencepiece.SentencePieceNormalizer__SetProtoField(self, name, value) + + def Init(self, + model_file=None, + model_proto=None, + rule_tsv=None, + rule_name=None, + add_dummy_prefix=False, + escape_whitespaces=False, + remove_extra_whitespaces=False): + """Initialzie sentencePieceNormalizer. + + Args: + model_file: The sentencepiece model file path. + model_proto: The sentencepiece model serialized proto. + rule_tsv: The normalization rule file in TSV format. + rule_name: Pre-defined normalization name. + add_dummy_prefix: add dummy prefix. + escape_whitespaces: escape whitespaces. + remove_extra_whitespaces: remove extra whitespaces. + """ + + _sentencepiece_normalizer_init_native(self) + + if model_file: + status = self.LoadFromFile(model_file) + elif model_proto: + status = self.LoadFromSerializedProto(model_proto) + elif rule_tsv: + status = self.LoadFromRuleTSV(rule_tsv) + elif rule_name: + status = self.LoadFromRuleName(rule_name) + else: + raise RuntimeError('no model is specified') + + if status: + self._SetProtoField('add_dummy_prefix', add_dummy_prefix) + self._SetProtoField('escape_whitespaces', escape_whitespaces) + self._SetProtoField('remove_extra_whitespaces', remove_extra_whitespaces) + + def Normalize(self, input, with_offsets=None): + def _normalize(text): + if with_offsets: + return self._NormalizeWithOffsets(text) + return self._Normalize(text) + + if type(input) is list: + return [_normalize(x) for x in input] + return _normalize(input) + + + def __getstate__(self): + return self.serialized_model_proto() + + + def __setstate__(self, serialized_model_proto): + self.__init__() + self.LoadFromSerializedProto(serialized_model_proto) + + +# Register SentencePieceNormalizer in _sentencepiece: +_sentencepiece.SentencePieceNormalizer_swigregister(SentencePieceNormalizer) + +def SetDataDir(data_dir): + return _sentencepiece.SetDataDir(data_dir) + + +import re +import csv +import sys +import os +import importlib.resources +from io import StringIO +from io import BytesIO + + +def _add_snake_case(classname): + """Added snake_cased method from CammelCased method.""" + + snake_map = {} + for k, v in classname.__dict__.items(): + if re.match(r'^[A-Z]+', k): + snake = re.sub(r'(?= v.piece_size()): + raise IndexError('piece id is out of range.') + return func(v, n) + + def _batched_func(self, arg): + if type(arg) is list: + return [_func(self, n) for n in arg] + else: + return _func(self, arg) + + setattr(classname, name, _batched_func) + + +_sentencepiece_processor_init_native = SentencePieceProcessor.__init__ +_sentencepiece_normalizer_init_native = SentencePieceNormalizer.__init__ +setattr(SentencePieceProcessor, '__init__', SentencePieceProcessor.Init) +setattr(SentencePieceNormalizer, '__init__', SentencePieceNormalizer.Init) + +SentencePieceProcessor.Tokenize = SentencePieceProcessor.Encode +SentencePieceProcessor.Detokenize = SentencePieceProcessor.Decode + +for m in [ + 'PieceToId', 'IdToPiece', 'GetScore', 'IsUnknown', 'IsControl', 'IsUnused', + 'IsByte' +]: + _batchnize(SentencePieceProcessor, m) + +_add_snake_case(SentencePieceProcessor) +_add_snake_case(SentencePieceTrainer) +_add_snake_case(SentencePieceNormalizer) +set_random_generator_seed = SetRandomGeneratorSeed +set_min_log_level = SetMinLogLevel + +from ._version import __version__ + +SetDataDir(os.path.join(str(importlib.resources.files('sentencepiece')), 'package_data')) + +class _LogStream(object): + def __init__(self, ostream=None): + self.ostream = ostream + if self.ostream is not None: + self.orig_stream_fileno = sys.stderr.fileno() + + def __enter__(self): + if self.ostream is not None: + self.orig_stream_dup = os.dup(self.orig_stream_fileno) + os.dup2(self.ostream.fileno(), self.orig_stream_fileno) + + def __exit__(self, type, value, traceback): + if self.ostream is not None: + os.close(self.orig_stream_fileno) + os.dup2(self.orig_stream_dup, self.orig_stream_fileno) + os.close(self.orig_stream_dup) + self.ostream.close() + + diff --git a/lib/python3.12/site-packages/sentencepiece/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/sentencepiece/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31ad77676acd2da58c178086fa690a0336bfa0ac Binary files /dev/null and b/lib/python3.12/site-packages/sentencepiece/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/sentencepiece/__pycache__/_version.cpython-312.pyc b/lib/python3.12/site-packages/sentencepiece/__pycache__/_version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a02c46c3474324d28f6148cb73eeba8e40ac0b6a Binary files /dev/null and b/lib/python3.12/site-packages/sentencepiece/__pycache__/_version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/sentencepiece/__pycache__/sentencepiece_model_pb2.cpython-312.pyc b/lib/python3.12/site-packages/sentencepiece/__pycache__/sentencepiece_model_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aef135733787c3d1902ac6e9ae80592df89af041 Binary files /dev/null and b/lib/python3.12/site-packages/sentencepiece/__pycache__/sentencepiece_model_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/sentencepiece/__pycache__/sentencepiece_pb2.cpython-312.pyc b/lib/python3.12/site-packages/sentencepiece/__pycache__/sentencepiece_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8d5942fdfe565c3b7dfa0b9110c2e045f5b7f425 Binary files /dev/null and b/lib/python3.12/site-packages/sentencepiece/__pycache__/sentencepiece_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/sentencepiece/_version.py b/lib/python3.12/site-packages/sentencepiece/_version.py new file mode 100644 index 0000000000000000000000000000000000000000..fc79d63d5430b972ac6ec1c4bfea9af80922da4d --- /dev/null +++ b/lib/python3.12/site-packages/sentencepiece/_version.py @@ -0,0 +1 @@ +__version__ = '0.2.1' diff --git a/lib/python3.12/site-packages/sentencepiece/sentencepiece.i b/lib/python3.12/site-packages/sentencepiece/sentencepiece.i new file mode 100644 index 0000000000000000000000000000000000000000..20944ea1ea2ade6d4a1b2debf4e90aa8e16d91d6 --- /dev/null +++ b/lib/python3.12/site-packages/sentencepiece/sentencepiece.i @@ -0,0 +1,2013 @@ +%module sentencepiece +%include exception.i + +%{ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +PyObject* kUnicodeInput = reinterpret_cast(0x1); +PyObject* kByteInput = reinterpret_cast(0x2); + +using BytesArray = std::vector; + +inline void ReleaseResultObject(PyObject *obj) { + if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) { + Py_XDECREF(obj); + } +} + +class PyInputString { + public: + explicit PyInputString(PyObject* obj) { + if (PyUnicode_Check(obj)) { + str_ = const_cast(PyUnicode_AsUTF8AndSize(obj, &size_)); + input_type_ = kUnicodeInput; + } else if (PyBytes_Check(obj)) { + PyBytes_AsStringAndSize(obj, &str_, &size_); + input_type_ = kByteInput; + } else { + str_ = nullptr; + } + } + absl::string_view str() const { return absl::string_view(data(), size()); } + const char* data() const { return str_; } + Py_ssize_t size() const { return size_; } + bool IsAvalable() const { return str_ != nullptr; } + PyObject *input_type() const { return input_type_; } + + static bool IsUnicode(PyObject *resultobj) { + return (resultobj == nullptr || resultobj == kUnicodeInput); + } + + private: + PyObject* input_type_ = nullptr; + char* str_ = nullptr; + Py_ssize_t size_ = 0; +}; + +PyObject* MakePyOutputString(const std::string& output, + PyObject *resultobj) { + if (PyInputString::IsUnicode(resultobj)) { + return PyUnicode_FromStringAndSize(output.data(), output.size()); + } + return PyBytes_FromStringAndSize(output.data(), output.size()); +} + +PyObject* MakePyOutputBytes(const sentencepiece::util::bytes& output) { + return PyBytes_FromStringAndSize(output.data(), output.size()); +} + +int ToSwigError(sentencepiece::util::StatusCode code) { + switch (code) { + case sentencepiece::util::StatusCode::kNotFound: + return SWIG_IOError; + case sentencepiece::util::StatusCode::kOutOfRange: + return SWIG_IndexError; + case sentencepiece::util::StatusCode::kInvalidArgument: + return SWIG_SyntaxError; + default: + return SWIG_RuntimeError; + } + return SWIG_RuntimeError; +} + +class PySentenceIterator : public sentencepiece::SentenceIterator { + public: + PySentenceIterator(PyObject *iter) : iter_(iter) { + item_ = PyIter_Next(iter_); + CopyValue(); + } + + ~PySentenceIterator() { + // Py_XDECREF(iter_); + } + + bool done() const override { + return item_ == nullptr; + } + + void Next() override { + item_ = PyIter_Next(iter_); + CopyValue(); + } + + const std::string &value() const override { + return value_; + } + + sentencepiece::util::Status status() const override { + return status_; + } + + private: + void CopyValue() { + if (item_ == nullptr) return; + const PyInputString ustring(item_); + if (ustring.IsAvalable()) { + const char *data = ustring.data(); + size_t size = ustring.size(); + while (size > 0) { + if (data[size - 1] == '\r' || data[size - 1] == '\n') + --size; + else + break; + } + value_.assign(data, size); + } else { + status_ = sentencepiece::util::Status(sentencepiece::util::StatusCode::kInternal, + "Not a string."); + } + Py_XDECREF(item_); + } + PyObject *iter_ = nullptr; + PyObject *item_ = nullptr; + std::string value_; + sentencepiece::util::Status status_; +}; + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + std::vector *ids, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (!add_bos && !add_eos && !reverse) return; + if (reverse) std::reverse(ids->begin(), ids->end()); + if (add_bos) ids->insert(ids->begin(), sp.bos_id()); + if (add_eos) ids->push_back(sp.eos_id()); +} + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + std::vector *pieces, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (!add_bos && !add_eos && !reverse && !emit_unk_piece) return; + if (reverse) std::reverse(pieces->begin(), pieces->end()); + if (add_bos) pieces->insert(pieces->begin(), sp.IdToPiece(sp.bos_id())); + if (add_eos) pieces->push_back(sp.IdToPiece(sp.eos_id())); + if (emit_unk_piece) { + const auto &unk = sp.IdToPiece(sp.unk_id()); + for (auto &piece : *pieces) { + const int id = sp.PieceToId(piece); + if (id == sp.unk_id()) { + piece = unk; + } + } + } +} + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + sentencepiece::util::bytes *proto, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (add_bos || add_eos || reverse || emit_unk_piece) { + throw sentencepiece::util::Status( + sentencepiece::util::StatusCode::kUnimplemented, + "add_bos, add_eos, reverse, and emit_unk_piece is not supported in proto API"); + } +} + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + sentencepiece::ImmutableSentencePieceText *proto, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (add_bos || add_eos || reverse || emit_unk_piece) { + throw sentencepiece::util::Status( + sentencepiece::util::StatusCode::kUnimplemented, + "add_bos, add_eos, reverse, and emit_unk_piece is not supported in proto API"); + } +} + +inline void CheckIds(const std::vector &ids, int num_pieces) { + for (int id : ids) { + if (id < 0 || id >= num_pieces) { + throw sentencepiece::util::Status( + sentencepiece::util::StatusCode::kOutOfRange, + "piece id is out of range."); + } + } +} + +inline void CheckIds(const std::vector &ids, int num_pieces) {} + +inline void CheckIdsBatch(const std::vector> &ids, int num_pieces) { + for (const auto &v : ids) CheckIds(v, num_pieces); +} + +template +inline void ConvertToUnicodeSpans(T *proto) {} + +template <> +inline void ConvertToUnicodeSpans(sentencepiece::ImmutableSentencePieceText *proto) { + proto->ConvertToUnicodeSpans(); +} + +template <> +inline void ConvertToUnicodeSpans(sentencepiece::ImmutableNBestSentencePieceText *proto) { + proto->ConvertToUnicodeSpans(); +} + +class ThreadPool { + public: + explicit ThreadPool(size_t request_size) : + request_size_(request_size) {} + + virtual ~ThreadPool() { + for (auto &task : tasks_) { + task.join(); + } + } + + void Schedule(std::function closure) { + static constexpr size_t kMinThreadSize = 2; + if (request_size_ < kMinThreadSize) { + closure(); + } else { + tasks_.emplace_back(closure); + } + } + + private: + size_t request_size_ = 0; + std::vector tasks_; +}; + +template +inline void InitNumThreads(const std::vector &ins, int *num_threads) { + if (*num_threads < 0) { + *num_threads = std::thread::hardware_concurrency(); + } + *num_threads = std::max(1, + std::min({*num_threads, + static_cast(ins.size()), 256})); +} + +#define DEFINE_ENCODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \ + std::vector outs(ins.size()); \ + InitNumThreads(ins, &num_threads); \ + { \ + ThreadPool pool(ins.size()); \ + std::atomic index = 0; \ + for (int n = 0; n < num_threads; ++n) { \ + pool.Schedule([&]() { \ + size_t i = 0; \ + while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { \ + auto out = enable_sampling ? \ + self->Sample##FuncName(ins[i], \ + nbest_size, alpha) : \ + self->FuncName(ins[i]); \ + RewriteIds(*self, &out, add_bos, add_eos, reverse, \ + emit_unk_piece); \ + ConvertToUnicodeSpans(&out); \ + outs[i] = std::move(out); \ + } \ + }); \ + } \ + } \ + return outs; + +#define DEFINE_DECODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \ + std::vector outs(ins.size()); \ + InitNumThreads(ins, &num_threads); \ + { \ + std::atomic index = 0; \ + ThreadPool pool(ins.size()); \ + for (int n = 0; n < num_threads; ++n) { \ + pool.Schedule([&]() { \ + size_t i = 0; \ + while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { \ + auto out = self->FuncName(ins[i]); \ + ConvertToUnicodeSpans(&out); \ + outs[i] = std::move(out); \ + } \ + }); \ + } \ + } \ + return outs; + +} // namespace +%} + +%init %{ +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif +%} + +%exception { + try { + $action + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } +} + +%apply unsigned int { uint32_t } + +%ignore sentencepiece::util::Status; +%ignore sentencepiece::util::StatusCode; +%ignore absl::string_view; +%ignore std::string_view; +%ignore sentencepiece::SentencePieceText; +%ignore sentencepiece::NormalizerSpec; +%ignore sentencepiece::TrainerSpec; +%ignore sentencepiece::SentencePieceProcessor::status; +%ignore sentencepiece::ImmutableSentencePieceText::mutable_proto; +%ignore sentencepiece::ImmutableSentencePieceText::pieces() const; +%ignore sentencepiece::ImmutableSentencePieceText::ConvertToUnicodeSpans; +%ignore sentencepiece::ImmutableNBestSentencePieceText::mutable_proto; +%ignore sentencepiece::ImmutableNBestSentencePieceText::nbests() const; +%ignore sentencepiece::ImmutableNBestSentencePieceText::ConvertToUnicodeSpans; + +%ignore sentencepiece::SentencePieceProcessor::Encode; +%ignore sentencepiece::SentencePieceProcessor::SampleEncode; +%ignore sentencepiece::SentencePieceProcessor::NBestEncode; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScore; +%ignore sentencepiece::SentencePieceProcessor::Decode; + +%ignore sentencepiece::SentencePieceProcessor::EncodeAsPieces; +%ignore sentencepiece::SentencePieceProcessor::EncodeAsIds; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsIds; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsPieces; +%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsIds; +%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsPieces; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsIds; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsPieces; +%ignore sentencepiece::SentencePieceProcessor::DecodeIds; +%ignore sentencepiece::SentencePieceProcessor::DecodePieces; + +%ignore sentencepiece::SentencePieceProcessor::EncodeAsSerializedProto; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsSerializedProto; +%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsSerializedProto; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsSerializedProto; +%ignore sentencepiece::SentencePieceProcessor::DecodePiecesAsSerializedProto; +%ignore sentencepiece::SentencePieceProcessor::DecodeIdsAsSerializedProto; + +%ignore sentencepiece::SentencePieceProcessor::EncodeAsImmutableProto; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAsImmutableProto; +%ignore sentencepiece::SentencePieceProcessor::NBestEncodeAsImmutableProto; +%ignore sentencepiece::SentencePieceProcessor::SampleEncodeAndScoreAsImmutableProto; +%ignore sentencepiece::SentencePieceProcessor::DecodePiecesAsImmutableProto; +%ignore sentencepiece::SentencePieceProcessor::DecodeIdsAsImmutableProto; + +%ignore sentencepiece::SentencePieceProcessor::Normalize; +%ignore sentencepiece::SentencePieceProcessor::NormalizeWithOffsets; + +%ignore sentencepiece::SentencePieceProcessor::model_proto; +%ignore sentencepiece::SentencePieceProcessor::mutable_normalizer_spec; +%ignore sentencepiece::SentencePieceProcessor::Load; +%ignore sentencepiece::SentencePieceProcessor::LoadOrDie; +%ignore sentencepiece::SentencePieceProcessor::SetModel; +%ignore sentencepiece::SentencePieceProcessor::SetNormalizer; +%ignore sentencepiece::pretokenizer::PretokenizerForTrainingInterface; +%ignore sentencepiece::SentenceIterator; +%ignore sentencepiece::ConvertToUnicodeSpans; +%ignore sentencepiece::SentencePieceTrainer::Train; +%ignore sentencepiece::SentencePieceTrainer::GetNormalizerSpec; +%ignore sentencepiece::SentencePieceTrainer::PopulateNormalizerSpec; +%ignore sentencepiece::SentencePieceTrainer::MergeSpecsFromArgs; +%ignore sentencepiece::SentencePieceTrainer::SetProtoField; +%ignore sentencepiece::SentencePieceTrainer::PopulateModelTypeFromString; +%ignore sentencepiece::SentencePieceTrainer::PieceProcecssor; +%ignore sentencepiece::SentencePieceTrainer::SetPretokenizerForTraining; +%ignore sentencepiece::SentencePieceTrainer::GetPretokenizerForTraining; +%ignore sentencepiece::SentencePieceTrainer::SetDataDir; +%ignore sentencepiece::ConvertToUnicodeAlignment; + +%ignore sentencepiece::SentencePieceNormalizer::Load; +%ignore sentencepiece::SentencePieceNormalizer::Normalize; +%ignore sentencepiece::SentencePieceNormalizer::mutable_normalizer_spec; + +%ignore sentencepiece::io::LoadModelProto; +%ignore sentencepiece::io::SaveModelProto; + +%extend sentencepiece::SentencePieceProcessor { + sentencepiece::util::Status LoadFromFile(absl::string_view arg) { + return $self->Load(arg); + } + + ///////////////////////////////////////////////////////////////////////////// + // EncodeAs* (Single request) + std::vector _EncodeAsIds(absl::string_view text, + bool enable_sampling, + int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto ids = enable_sampling ? + $self->SampleEncodeAsIds(text, nbest_size, alpha) : + $self->EncodeAsIds(text); + RewriteIds(*$self, &ids, add_bos, add_eos, reverse, emit_unk_piece); + return ids; + } + + std::vector _EncodeAsPieces(absl::string_view text, + bool enable_sampling, + int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto pieces = enable_sampling ? + $self->SampleEncodeAsPieces(text, nbest_size, alpha) : + $self->EncodeAsPieces(text); + RewriteIds(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece); + return pieces; + } + + sentencepiece::util::bytes _EncodeAsSerializedProto(absl::string_view text, + bool enable_sampling, + int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto proto = enable_sampling ? + $self->SampleEncodeAsSerializedProto(text, nbest_size, alpha) : + $self->EncodeAsSerializedProto(text); + RewriteIds(*$self, &proto, add_bos, add_eos, reverse, emit_unk_piece); + return proto; + } + + sentencepiece::ImmutableSentencePieceText + _EncodeAsImmutableProto(absl::string_view text, + bool enable_sampling, + int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto proto = enable_sampling ? + $self->SampleEncodeAsImmutableProto(text, nbest_size, alpha) : + $self->EncodeAsImmutableProto(text); + proto.ConvertToUnicodeSpans(); + RewriteIds(*$self, &proto, add_bos, add_eos, reverse, emit_unk_piece); + return proto; + } + + ///////////////////////////////////////////////////////////////////////////// + // EncodeAs* (Batch request) + std::vector> _EncodeAsIdsBatch( + const std::vector &ins, int num_threads, + bool enable_sampling, int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsIds, + absl::string_view, std::vector); + } + + std::vector> _EncodeAsPiecesBatch( + const std::vector &ins, int num_threads, + bool enable_sampling, int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsPieces, + absl::string_view, std::vector); + } + + BytesArray _EncodeAsSerializedProtoBatch( + const std::vector &ins, int num_threads, + bool enable_sampling, int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsSerializedProto, + absl::string_view, + sentencepiece::util::bytes); + } + + std::vector + _EncodeAsImmutableProtoBatch( + const std::vector &ins, int num_threads, + bool enable_sampling, int nbest_size, float alpha, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsImmutableProto, + absl::string_view, + sentencepiece::ImmutableSentencePieceText); + } + + ///////////////////////////////////////////////////////////////////////////// + // DecodeAs* (Single request) + std::string _DecodeIds(const std::vector &ids) const { + CheckIds(ids, $self->GetPieceSize()); + return $self->DecodeIds(ids); + } + + sentencepiece::util::bytes _DecodeIdsAsBytes(const std::vector &ids) const { + CheckIds(ids, $self->GetPieceSize()); + return $self->DecodeIds(ids); + } + + std::string _DecodePieces(const std::vector &pieces) const { + return $self->DecodePieces(pieces); + } + + sentencepiece::util::bytes _DecodeIdsAsSerializedProto( + const std::vector &ids) const { + CheckIds(ids, $self->GetPieceSize()); + return $self->DecodeIdsAsSerializedProto(ids); + } + + sentencepiece::util::bytes _DecodePiecesAsSerializedProto( + const std::vector &pieces) const { + CheckIds(pieces, $self->GetPieceSize()); + return $self->DecodePiecesAsSerializedProto(pieces); + } + + sentencepiece::ImmutableSentencePieceText _DecodeIdsAsImmutableProto( + const std::vector &ids) const { + CheckIds(ids, $self->GetPieceSize()); + auto proto = $self->DecodeIdsAsImmutableProto(ids); + proto.ConvertToUnicodeSpans(); + return proto; + } + + sentencepiece::ImmutableSentencePieceText _DecodePiecesAsImmutableProto( + const std::vector &pieces) const { + CheckIds(pieces, $self->GetPieceSize()); + auto proto= $self->DecodePiecesAsImmutableProto(pieces); + proto.ConvertToUnicodeSpans(); + return proto; + } + + ///////////////////////////////////////////////////////////////////////////// + // DecodeAs* (Batch request) + std::vector _DecodeIdsBatch( + const std::vector> &ins, int num_threads) const { + CheckIdsBatch(ins, $self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string); + } + + BytesArray _DecodeIdsAsBytesBatch( + const std::vector> &ins, int num_threads) const { + CheckIdsBatch(ins, $self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string); + } + + BytesArray _DecodeIdsAsSerializedProtoBatch( + const std::vector> &ins, int num_threads) const { + CheckIdsBatch(ins, $self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsSerializedProto, int, + sentencepiece::util::bytes); + } + + std::vector + _DecodeIdsAsImmutableProtoBatch( + const std::vector> &ins, int num_threads) const { + CheckIdsBatch(ins, $self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsImmutableProto, int, + sentencepiece::ImmutableSentencePieceText); + } + + std::vector _DecodePiecesBatch( + const std::vector> &ins, int num_threads) const { + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePieces, std::string, std::string); + } + + BytesArray _DecodePiecesAsSerializedProtoBatch( + const std::vector> &ins, int num_threads) const { + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsSerializedProto, std::string, + sentencepiece::util::bytes); + } + + std::vector + _DecodePiecesAsImmutableProtoBatch( + const std::vector> &ins, int num_threads) const { + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsImmutableProto, std::string, + sentencepiece::ImmutableSentencePieceText); + } + + //////////////////////////////////////////////////////////////////////////// + // NBestEncodeAs* (Single request) + std::vector> + _NBestEncodeAsIds(absl::string_view text, + int nbest_size, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto idss = $self->NBestEncodeAsIds(text, nbest_size); + for (auto &ids : idss) { + RewriteIds(*$self, &ids, add_bos, add_eos, reverse, emit_unk_piece); + } + return idss; + } + + std::vector> + _NBestEncodeAsPieces(absl::string_view text, + int nbest_size, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto piecess = $self->NBestEncodeAsPieces(text, nbest_size); + for (auto &pieces : piecess) { + RewriteIds(*$self, &pieces, add_bos, add_eos, reverse, emit_unk_piece); + } + return piecess; + } + + sentencepiece::util::bytes + _NBestEncodeAsSerializedProto(absl::string_view text, + int nbest_size, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + RewriteIds(*$self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + return $self->NBestEncodeAsSerializedProto(text, nbest_size); + } + + sentencepiece::ImmutableNBestSentencePieceText + _NBestEncodeAsImmutableProto(absl::string_view text, + int nbest_size, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + RewriteIds(*$self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + auto proto = $self->NBestEncodeAsImmutableProto(text, nbest_size); + proto.ConvertToUnicodeSpans(); + return proto; + } + + + ///////////////////////////////////////////////////////////////////////////// + // SampleEncodeAndScoreAs* (Single request) + std::vector, float>> + _SampleEncodeAndScoreAsIds(absl::string_view text, + int num_samples, float alpha, bool wor, + bool include_best, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto idss = $self->SampleEncodeAndScoreAsIds(text, num_samples, + alpha, wor, include_best); + for (auto &ids : idss) { + RewriteIds(*$self, &ids.first, add_bos, add_eos, reverse, emit_unk_piece); + } + return idss; + } + + std::vector, float>> + _SampleEncodeAndScoreAsPieces(absl::string_view text, + int num_samples, float alpha, bool wor, + bool include_best, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + auto piecess = $self->SampleEncodeAndScoreAsPieces(text, num_samples, + alpha, wor, include_best); + for (auto &pieces : piecess) { + RewriteIds(*$self, &pieces.first, add_bos, add_eos, reverse, emit_unk_piece); + } + return piecess; + } + + sentencepiece::util::bytes + _SampleEncodeAndScoreAsSerializedProto(absl::string_view text, + int num_samples, float alpha, bool wor, + bool include_best, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + RewriteIds(*$self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + return $self->SampleEncodeAndScoreAsSerializedProto(text, num_samples, + alpha, wor, include_best); + } + + sentencepiece::ImmutableNBestSentencePieceText + _SampleEncodeAndScoreAsImmutableProto(absl::string_view text, + int num_samples, float alpha, bool wor, + bool include_best, + bool add_bos, bool add_eos, bool reverse, + bool emit_unk_piece) const { + RewriteIds(*$self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + auto proto = $self->SampleEncodeAndScoreAsImmutableProto(text, num_samples, + alpha, wor, include_best); + proto.ConvertToUnicodeSpans(); + return proto; + } + + // Normalize + std::string _Normalize(absl::string_view text) { + return $self->Normalize(text); + } + + std::pair> _NormalizeWithOffsets(absl::string_view text) { + std::pair> result; + $self->Normalize(text, &result.first, &result.second).IgnoreError(); + return result; + } + + // Calculate Entropy + float _CalculateEntropy(absl::string_view text, float alpha) { + return $self->CalculateEntropy(text, alpha); + } + + std::vector _CalculateEntropyBatch(const std::vector &ins, + float alpha, int num_threads) { + std::vector outs(ins.size()); + InitNumThreads(ins, &num_threads); + { + ThreadPool pool(ins.size()); + std::atomic index = 0; + for (int n = 0; n < num_threads; ++n) { + pool.Schedule([&]() { + size_t i = 0; + while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { + outs[i] = self->CalculateEntropy(ins[i], alpha); + } + }); + } + } + return outs; + } + + // override normalizer_spec + sentencepiece::util::Status _OverrideNormalizerSpec( + const std::unordered_map &args) { + sentencepiece::util::Status status; + for (const auto &[key, value] : args) { + status = sentencepiece::SentencePieceTrainer::SetProtoField( + key, value, + $self->mutable_normalizer_spec()); + if (!status.ok()) return status; + } + return status; + } + +%pythoncode { + def Init(self, + model_file=None, + model_proto=None, + out_type=int, + add_bos=False, + add_eos=False, + reverse=False, + emit_unk_piece=False, + enable_sampling=False, + nbest_size=-1, + alpha=0.1, + num_threads=-1): + """Initialzie sentencepieceProcessor. + + Args: + model_file: The sentencepiece model file path. + model_proto: The sentencepiece model serialized proto. + out_type: output type. int or str. + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after + reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. + nbest_size = {0,1}: No sampling is performed. + nbest_size > 1: samples from the nbest_size results. + nbest_size < 0: assuming that nbest_size is infinite and samples + from the all hypothesis (lattice) using + forward-filtering-and-backward-sampling algorithm. + alpha: Soothing parameter for unigram sampling, and dropout probability of + merge operations for BPE-dropout. + num_threads: number of threads in batch processing (Default = -1, auto-detected) + """ + + _sentencepiece_processor_init_native(self) + self._out_type = out_type + self._add_bos = add_bos + self._add_eos = add_eos + self._reverse = reverse + self._emit_unk_piece = emit_unk_piece + self._enable_sampling = enable_sampling + self._nbest_size = nbest_size + self._alpha = alpha + self._num_threads = num_threads + if model_file or model_proto: + self.Load(model_file=model_file, model_proto=model_proto) + + + def Encode(self, + input, + out_type=None, + add_bos=None, + add_eos=None, + reverse=None, + emit_unk_piece=None, + enable_sampling=None, + nbest_size=None, + alpha=None, + num_threads=None): + """Encode text input to segmented ids or tokens. + + Args: + input: input string. accepsts list of string. + out_type: output type. int or str. + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after + reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + nbest_size: sampling parameters for unigram. Invalid in BPE-Dropout. + nbest_size = {0,1}: No sampling is performed. + nbest_size > 1: samples from the nbest_size results. + nbest_size < 0: assuming that nbest_size is infinite and samples + from the all hypothesis (lattice) using + forward-filtering-and-backward-sampling algorithm. + alpha: Soothing parameter for unigram sampling, and merge probability for + BPE-dropout (probablity 'p' in BPE-dropout paper). + num_threads: the number of threads used in the batch processing (Default = -1). + """ + + if out_type is None: + out_type = self._out_type + if add_bos is None: + add_bos = self._add_bos + if add_eos is None: + add_eos = self._add_eos + if reverse is None: + reverse = self._reverse + if emit_unk_piece is None: + emit_unk_piece = self._emit_unk_piece + if enable_sampling is None: + enable_sampling = self._enable_sampling + if nbest_size is None: + nbest_size = self._nbest_size + if alpha is None: + alpha = self._alpha + if num_threads is None: + num_threads = self._num_threads + + if enable_sampling == True and (nbest_size is None or nbest_size == 0 or + nbest_size == 1 or alpha is None): + raise RuntimeError( + 'When enable_sampling is True, We must specify "nbest_size > 1" or "nbest_size = -1", ' + 'and "alpha". "nbest_size" is enabled only on unigram mode ignored in BPE-dropout. ' + 'when "nbest_size = -1" , this method samples from all candidates on the lattice ' + 'instead of nbest segmentations.' + ) + + if num_threads is None or type(num_threads) is not int: + raise RuntimeError('num_threads must be int') + + if type(input) is list: + if out_type is int: + return self._EncodeAsIdsBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._EncodeAsPiecesBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'serialized_proto' or out_type == 'proto': + return self._EncodeAsSerializedProtoBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'immutable_proto': + return self._EncodeAsImmutableProtoBatch(input, num_threads, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + + if out_type is int: + return self._EncodeAsIds(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._EncodeAsPieces(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'serialized_proto' or out_type == 'proto': + return self._EncodeAsSerializedProto(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'immutable_proto': + return self._EncodeAsImmutableProto(input, enable_sampling, nbest_size, + alpha, add_bos, add_eos, reverse, emit_unk_piece) + + raise RuntimeError('unknown out_type={}'.format(out_type)) + return None + + + def EncodeAsPieces(self, input, **kwargs): + return self.Encode(input=input, out_type=str, **kwargs) + + + def EncodeAsIds(self, input, **kwargs): + return self.Encode(input=input, out_type=int, **kwargs) + + + def EncodeAsSerializedProto(self, input, **kwargs): + return self.Encode(input=input, out_type='serialized_proto', **kwargs) + + + def EncodeAsImmutableProto(self, input, **kwargs): + return self.Encode(input=input, out_type='immutable_proto', **kwargs) + + + def SampleEncodeAsPieces(self, input, nbest_size=None, alpha=None, **kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type=str, enable_sampling=True, **kwargs) + + + def SampleEncodeAsIds(self, input, nbest_size=None, alpha=None,**kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type=int, enable_sampling=True, **kwargs) + + + def SampleEncodeAsSerializedProto(self, input, nbest_size=None, alpha=None, **kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type='serialized_proto', enable_sampling=True, **kwargs) + + + def SampleEncodeAsImmutableProto(self, input, nbest_size=None, alpha=None, **kwargs): + return self.Encode(input=input, nbest_size=nbest_size, alpha=alpha, + out_type='immutable_proto', enable_sampling=True, **kwargs) + + + def NBestEncode(self, + input, + out_type=None, + add_bos=None, + add_eos=None, + reverse=None, + emit_unk_piece=None, + nbest_size=None): + """NBestEncode text input to segmented ids or tokens. + + Args: + input: input string. accepsts list of string. + out_type: output type. int or str. + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + nbest_size: nbest size + """ + + if out_type is None: + out_type = self._out_type + if add_bos is None: + add_bos = self._add_bos + if add_eos is None: + add_eos = self._add_eos + if reverse is None: + reverse = self._reverse + if emit_unk_piece is None: + emit_unk_piece = self._emit_unk_piece + if nbest_size is None: + nbest_size = self._nbest_size + + if nbest_size <= 0: + nbest_size=1 + + def _encode(text): + if out_type is int: + return self._NBestEncodeAsIds(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._NBestEncodeAsPieces(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'serialized_proto' or out_type == 'proto': + return self._NBestEncodeAsSerializedProto(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type == 'immutable_proto': + return self._NBestEncodeAsImmutableProto(text, nbest_size, + add_bos, add_eos, reverse, emit_unk_piece) + + raise RuntimeError('unknown out_type') + + if type(input) is list: + return [_encode(n) for n in input] + + return _encode(input) + + + def NBestEncodeAsPieces(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type=str, **kwargs) + + + def NBestEncodeAsIds(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type=int, **kwargs) + + + def NBestEncodeAsSerializedProto(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type='serialized_proto', **kwargs) + + + def NBestEncodeAsImmutableProto(self, input, nbest_size=None, **kwargs): + return self.NBestEncode(input=input, nbest_size=nbest_size, + out_type='immutable_proto', **kwargs) + + + def SampleEncodeAndScore(self, + input, + out_type=None, + add_bos=None, + add_eos=None, + reverse=None, + emit_unk_piece=None, + num_samples=None, + alpha=None, + wor=None, + include_best=None): + """SampleEncodeAndScore text input to segmented ids or tokens. + + Args: + input: input string. accepsts list of string. + out_type: output type. int or str or 'serialized_proto' or 'immutable_proto' + add_bos: Add to the result (Default = false) + add_eos: Add to the result (Default = false) / is added after reversing (if enabled). + reverse: Reverses the tokenized sequence (Default = false) + emit_unk_piece: Emits the unk literal string (Default = false) + num_samples: How many samples to return (Default = 1) + alpha: inverse temperature for sampling + wor: whether to sample without replacement (Default = false) + include_best: whether to include the best tokenization, requires wor=True (Default = false) + """ + + if out_type is None: + out_type = self._out_type + if add_bos is None: + add_bos = self._add_bos + if add_eos is None: + add_eos = self._add_eos + if reverse is None: + reverse = self._reverse + if emit_unk_piece is None: + emit_unk_piece = self._emit_unk_piece + if num_samples is None: + num_samples = 1 + if alpha is None: + alpha = 1. + if wor is None: + wor = False + if include_best is None: + include_best = False + + if num_samples <= 0: + raise RuntimeError('num_examples must be positive') + + if include_best and not wor: + raise RuntimeError('When include_best is True, We must specify "wor = True".') + + + def _encode(text): + if out_type is int: + return self._SampleEncodeAndScoreAsIds(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + if out_type is str: + return self._SampleEncodeAndScoreAsPieces(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + + if out_type == 'serialized_proto' or out_type == 'proto': + return self._SampleEncodeAndScoreAsSerializedProto(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + + if out_type == 'immutable_proto': + return self._SampleEncodeAndScoreAsImmutableProto(text, num_samples, alpha, wor, include_best, + add_bos, add_eos, reverse, emit_unk_piece) + + raise RuntimeError('unknown output type') + + + if type(input) is list: + return [_encode(n) for n in input] + + return _encode(input) + + + def SampleEncodeAndScoreAsPieces(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type=str, **kwargs) + + + def SampleEncodeAndScoreAsIds(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type=int, **kwargs) + + + def SampleEncodeAndScoreAsSerializedProto(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type='serialized_proto', **kwargs) + + + def SampleEncodeAndScoreAsImmutableProto(self, input, num_samples=None, alpha=None, **kwargs): + return self.SampleEncodeAndScore(input=input, num_samples=num_samples, alpha=alpha, + out_type='immutable_proto', **kwargs) + + + def Decode(self, input, out_type=str, num_threads=None): + """Decode processed id or token sequences. + + Args: + out_type: output type. str, bytes or 'serialized_proto' or 'immutable_proto' (Default = str) + num_threads: the number of threads used in the batch processing (Default = -1). + """ + + if num_threads is None: + num_threads = self._num_threads + + if num_threads is None or type(num_threads) is not int: + raise RuntimeError('num_threads must be int') + + if not input: + return '' + + if out_type is str: + if type(input) is int: + return self._DecodeIds([input]) + if type(input) is str: + return self._DecodePieces([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIds(input) + if type(input[0]) is str: + return self._DecodePieces(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesBatch(input, num_threads) + + if out_type is bytes: + if type(input) is int: + return self._DecodeIdsAsBytes([input]) + if type(input) is str: + return self._DecodePieces([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIdsAsBytes(input) + if type(input[0]) is str: + return self._DecodePieces(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsAsBytesBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesBatch(input, num_threads) + + if out_type == 'serialized_proto': + if type(input) is int: + return self._DecodeIdsAsSerializedProto([input]) + if type(input) is str: + return self._DecodePiecesAsSerializedProto([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIdsAsSerializedProto(input) + if type(input[0]) is str: + return self._DecodePiecesAsSerializedProto(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsAsSerializedProtoBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesAsSerializedProtoBatch(input, num_threads) + + + if out_type == 'immutable_proto': + if type(input) is int: + return self._DecodeIdsAsImmutableProto([input]) + if type(input) is str: + return self._DecodePiecesAsImmutableProto([input]) + + if type(input) is list: + if len(input) == 0 or type(input[0]) is int: + return self._DecodeIdsAsImmutableProto(input) + if type(input[0]) is str: + return self._DecodePiecesAsImmutableProto(input) + + if type(input[0]) is list: + if len(input[0]) == 0 or type(input[0][0]) is int: + return self._DecodeIdsAsImmutableProtoBatch(input, num_threads) + if type(input[0][0]) is str: + return self._DecodePiecesAsImmutableProtoBatch(input, num_threads) + + + raise RuntimeError('unknown output or input type') + return None + + + def DecodePieces(self, input, out_type=str, **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodeIds(self, input, out_type=str, **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodePiecesAsSerializedProto(self, input, out_type='serialized_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodeIdsAsSerializedProto(self, input, out_type='serialized_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodePiecesAsImmutableProto(self, input, out_type='immutable_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def DecodeIdsAsImmutableProto(self, input, out_type='immutable_proto', **kwargs): + return self.Decode(input=input, out_type=out_type, **kwargs) + + + def CalculateEntropy(self, input, alpha, num_threads=None): + """Calculate sentence entropy""" + if type(input) is list: + if num_threads is None: + num_threads = self._num_threads + if num_threads is None or type(num_threads) is not int: + raise RuntimeError('num_threads must be int') + return self._CalculateEntropyBatch(input, alpha, num_threads) + + return self._CalculateEntropy(input, alpha) + + + def Normalize(self, input, with_offsets=None): + def _normalize(text): + if with_offsets: + return self._NormalizeWithOffsets(text) + return self._Normalize(text) + + if type(input) is list: + return [_normalize(x) for x in input] + return _normalize(input) + + def OverrideNormalizerSpec(self, **kwargs): + new_kwargs = {} + for key, value in kwargs.items(): + new_kwargs[key] = str(value) + return self._OverrideNormalizerSpec(new_kwargs) + + + def piece_size(self): + return self.GetPieceSize() + + + def vocab_size(self): + return self.GetPieceSize() + + + def __getstate__(self): + return self.serialized_model_proto() + + + def __setstate__(self, serialized_model_proto): + self.__init__() + self.LoadFromSerializedProto(serialized_model_proto) + + + def __len__(self): + return self.GetPieceSize() + + + def __getitem__(self, piece): + return self.PieceToId(piece) + + + def Load(self, model_file=None, model_proto=None): + """Overwride SentencePieceProcessor.Load to support both model_file and model_proto. + + Args: + model_file: The sentencepiece model file path. + model_proto: The sentencepiece model serialized proto. Either `model_file` + or `model_proto` must be set. + """ + if model_file and model_proto: + raise RuntimeError('model_file and model_proto must be exclusive.') + if model_proto: + return self.LoadFromSerializedProto(model_proto) + return self.LoadFromFile(model_file) +} +} + +%extend sentencepiece::SentencePieceTrainer { + static void _TrainFromString(absl::string_view arg) { + const auto _status = sentencepiece::SentencePieceTrainer::Train(arg); + if (!_status.ok()) throw _status; + return; + } + + static void _TrainFromMap(const std::unordered_map &args) { + const auto _status = sentencepiece::SentencePieceTrainer::Train(args); + if (!_status.ok()) throw _status; + return; + } + + static void _TrainFromMap2(const std::unordered_map &args, + SentenceIterator *iter) { + const auto _status = sentencepiece::SentencePieceTrainer::Train(args, iter); + if (!_status.ok()) throw _status; + return; + } + + static sentencepiece::util::bytes _TrainFromMap3(const std::unordered_map &args) { + sentencepiece::util::bytes model_proto; + const auto _status = sentencepiece::SentencePieceTrainer::Train(args, nullptr, &model_proto); + if (!_status.ok()) throw _status; + return model_proto; + } + + static sentencepiece::util::bytes _TrainFromMap4(const std::unordered_map &args, + SentenceIterator *iter) { + sentencepiece::util::bytes model_proto; + const auto _status = sentencepiece::SentencePieceTrainer::Train(args, iter, &model_proto); + if (!_status.ok()) throw _status; + return model_proto; + } + +%pythoncode { + @staticmethod + def _Train(arg=None, **kwargs): + """Train Sentencepiece model. Accept both kwargs and legacy string arg.""" + if arg is not None and type(arg) is str: + return SentencePieceTrainer._TrainFromString(arg) + + def _encode(value): + """Encode value to CSV..""" + if type(value) is list: + if sys.version_info[0] == 3: + f = StringIO() + else: + f = BytesIO() + writer = csv.writer(f, lineterminator='') + writer.writerow([str(v) for v in value]) + return f.getvalue() + else: + return str(value) + + sentence_iterator = None + model_writer = None + new_kwargs = {} + for key, value in kwargs.items(): + if key in ['sentence_iterator', 'sentence_reader']: + sentence_iterator = value + elif key in ['model_writer']: + model_writer = value + else: + new_kwargs[key] = _encode(value) + + if model_writer: + if sentence_iterator: + model_proto = SentencePieceTrainer._TrainFromMap4(new_kwargs, + sentence_iterator) + else: + model_proto = SentencePieceTrainer._TrainFromMap3(new_kwargs) + model_writer.write(model_proto) + else: + if sentence_iterator: + return SentencePieceTrainer._TrainFromMap2(new_kwargs, sentence_iterator) + else: + return SentencePieceTrainer._TrainFromMap(new_kwargs) + + return None + + @staticmethod + def Train(arg=None, logstream=None, **kwargs): + with _LogStream(ostream=logstream): + SentencePieceTrainer._Train(arg=arg, **kwargs) +} +} + +%extend sentencepiece::SentencePieceNormalizer { + sentencepiece::util::Status LoadFromFile(absl::string_view arg) { + return $self->Load(arg); + } + + std::string _Normalize(absl::string_view text) { + std::string result; + const auto _status = $self->Normalize(text, &result); + if (!_status.ok()) throw _status; + return result; + } + + std::pair> _NormalizeWithOffsets(absl::string_view text) { + std::pair> result; + const auto _status = $self->Normalize(text, &result.first, &result.second); + if (!_status.ok()) throw _status; + return result; + } + + void _SetProtoField(absl::string_view name, bool value) { + sentencepiece::SentencePieceTrainer::SetProtoField( + name, + value ? "1" : "0", + $self->mutable_normalizer_spec()).IgnoreError(); + } + +%pythoncode %{ + def Init(self, + model_file=None, + model_proto=None, + rule_tsv=None, + rule_name=None, + add_dummy_prefix=False, + escape_whitespaces=False, + remove_extra_whitespaces=False): + """Initialzie sentencePieceNormalizer. + + Args: + model_file: The sentencepiece model file path. + model_proto: The sentencepiece model serialized proto. + rule_tsv: The normalization rule file in TSV format. + rule_name: Pre-defined normalization name. + add_dummy_prefix: add dummy prefix. + escape_whitespaces: escape whitespaces. + remove_extra_whitespaces: remove extra whitespaces. + """ + + _sentencepiece_normalizer_init_native(self) + + if model_file: + status = self.LoadFromFile(model_file) + elif model_proto: + status = self.LoadFromSerializedProto(model_proto) + elif rule_tsv: + status = self.LoadFromRuleTSV(rule_tsv) + elif rule_name: + status = self.LoadFromRuleName(rule_name) + else: + raise RuntimeError('no model is specified') + + if status: + self._SetProtoField('add_dummy_prefix', add_dummy_prefix) + self._SetProtoField('escape_whitespaces', escape_whitespaces) + self._SetProtoField('remove_extra_whitespaces', remove_extra_whitespaces) + + def Normalize(self, input, with_offsets=None): + def _normalize(text): + if with_offsets: + return self._NormalizeWithOffsets(text) + return self._Normalize(text) + + if type(input) is list: + return [_normalize(x) for x in input] + return _normalize(input) + + + def __getstate__(self): + return self.serialized_model_proto() + + + def __setstate__(self, serialized_model_proto): + self.__init__() + self.LoadFromSerializedProto(serialized_model_proto) +%} +} + +%extend sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece { + const sentencepiece::util::bytes& _surface_as_bytes() const { + return $self->surface(); + } + + const sentencepiece::util::bytes& _piece_as_bytes() const { + return $self->piece(); + } + + %rename(_piece) piece; + %rename(_piece_as_bytes) piece_as_bytes; + %rename(_id) id; + %rename(_surface) surface; + %rename(_surface_as_bytes) surface_as_bytes; + %rename(_begin) begin; + %rename(_end) end; + + %pythoncode %{ + piece = property(_piece) + piece_as_bytes = property(_piece_as_bytes) + surface = property(_surface) + surface_as_bytes = property(_surface_as_bytes) + id = property(_id) + begin = property(_begin) + end = property(_end) + + def __str__(self): + return ('piece: \"{}\"\n' + 'id: {}\n' + 'surface: \"{}\"\n' + 'begin: {}\n' + 'end: {}\n').format(self.piece, self.id, self.surface, + self.begin, self.end) + + def __eq__(self, other): + return self.piece == other.piece and self.id == other.id and self.surface == other.surface and self.begin == other.begin and self.end == other.end + + def __hash__(self): + return hash(str(self)) + + __repr__ = __str__ + %} +} + +%extend sentencepiece::ImmutableSentencePieceText { + const sentencepiece::util::bytes& _text_as_bytes() const { + return $self->text(); + } + + %rename(_text) text; + %rename(_text_as_bytes) text_as_bytes; + %rename(_score) score; + %rename(_pieces) pieces; + %rename(_pieces_size) pieces_size; + + %pythoncode %{ + text = property(_text) + text_as_bytes = property(_text_as_bytes) + score = property(_score) + + class ImmutableSentencePieceIterator: + def __init__(self, proto): + self.proto = proto + self.len = self.proto._pieces_size() + + def __len__(self): + return self.len + + def __getitem__(self, index): + if isinstance(index, slice): + return [self.proto._pieces(i) for i in range(self.len)][index.start:index.stop:index.step] + if index < 0: + index = index + self.len + if index < 0 or index >= self.len: + raise IndexError('piece index is out of range') + return self.proto._pieces(index) + + def __str__(self): + return '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self]) + + __repr__ = __str__ + + @property + def pieces(self): + return ImmutableSentencePieceText.ImmutableSentencePieceIterator(self) + + def __eq__(self, other): + return self.SerializeAsString() == other.SerializeAsString() + + def __hash__(self): + return hash(self.SerializeAsString()) + + def __str__(self): + return ('text: \"{}\"\n' + 'score: {}\n' + '{}').format(self.text, self.score, + '\n'.join(['pieces {{\n{}}}'.format(str(x)) for x in self.pieces])) + + __repr__ = __str__ + %} +} + +%extend sentencepiece::ImmutableNBestSentencePieceText { + %rename(_nbests) nbests; + %rename(_nbests_size) nbests_size; + + %pythoncode %{ + class ImmutableSentencePieceTextIterator: + def __init__(self, proto): + self.proto = proto + self.len = self.proto._nbests_size() + + def __len__(self): + return self.len + + def __getitem__(self, index): + if isinstance(index, slice): + return [self.proto._nbests(i) for i in range(self.len)][index.start:index.stop:index.step] + if index < 0: + index = index + self.len + if index < 0 or index >= self.len: + raise IndexError('nbests index is out of range') + return self.proto._nbests(index) + + def __str__(self): + return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self]) + + __repr__ = __str__ + + @property + def nbests(self): + return ImmutableNBestSentencePieceText.ImmutableSentencePieceTextIterator(self) + + def __eq__(self, other): + return self.SerializeAsString() == other.SerializeAsString() + + def __hash__(self): + return hash(self.SerializeAsString()) + + def __str__(self): + return '\n'.join(['nbests {{\n{}}}'.format(str(x)) for x in self.nbests]) + + __repr__ = __str__ + %} +} + +%typemap(out) std::vector { + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyList_SET_ITEM($result, i, PyInt_FromLong(static_cast($1[i]))); + } +} + +%typemap(out) std::vector { + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyList_SET_ITEM($result, i, PyFloat_FromDouble(static_cast($1[i]))); + } +} + +%typemap(out) std::vector> { + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyObject *obj = PyList_New($1[i].size()); + for (size_t j = 0; j < $1[i].size(); ++j) { + PyList_SET_ITEM(obj, j, PyInt_FromLong(static_cast($1[i][j]))); + } + PyList_SET_ITEM($result, i, obj); + } +} + +%typemap(out) std::vector { + PyObject *input_type = resultobj; + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyList_SET_ITEM($result, i, MakePyOutputString($1[i], input_type)); + } +} + +%typemap(out) BytesArray { + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyList_SET_ITEM($result, i, MakePyOutputBytes($1[i])); + } +} + +%typemap(out) std::vector> { + PyObject *input_type = resultobj; + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyObject *obj = PyList_New($1[i].size()); + for (size_t j = 0; j < $1[i].size(); ++j) { + PyList_SET_ITEM(obj, j, MakePyOutputString($1[i][j], input_type)); + } + PyList_SET_ITEM($result, i, obj); + } +} + +%typemap(out) sentencepiece::util::bytes { + $result = MakePyOutputBytes($1); +} + +%typemap(out) const sentencepiece::util::bytes& { + $result = MakePyOutputBytes(*$1); +} + +%typemap(out) std::string { + PyObject *input_type = resultobj; + $result = MakePyOutputString($1, input_type); +} + +%typemap(out) const std::string& { + PyObject *input_type = resultobj; + $result = MakePyOutputString(*$1, input_type); +} + +%typemap(out) sentencepiece::util::Status { + if (!$1.ok()) { + SWIG_exception(ToSwigError($1.code()), $1.ToString().c_str()); + } + $result = SWIG_From_bool($1.ok());} + + +%typemap(in) const std::string & { + const PyInputString ustring($input); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + $1 = new std::string(ustring.data(), ustring.size()); +} + +%typemap(typecheck) absl::string_view = char *; + +%typemap(in) absl::string_view { + const PyInputString ustring($input); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + $1 = ustring.str(); +} + +%typemap(in) const std::vector& { + std::vector *out = nullptr; + if (PyList_Check($input)) { + const size_t size = PyList_Size($input); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem($input, i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + $1 = out; +} + +%typemap(in) const std::vector& { + std::vector *out = nullptr; + if (PyList_Check($input)) { + const size_t size = PyList_Size($input); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem($input, i); + if (PyInt_Check(o)) { + (*out)[i] = static_cast(PyInt_AsLong(o)); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + $1 = out; +} + +%typemap(in) const std::vector>& { + std::vector> *out = nullptr; + if (PyList_Check($input)) { + const size_t size = PyList_Size($input); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem($input, i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + const PyInputString ustring(PyList_GetItem(o, j)); + if (ustring.IsAvalable()) { + (*out)[i][j] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + $1 = out; +} + +%typemap(in) const std::vector>& { + std::vector> *out = nullptr; + if (PyList_Check($input)) { + const size_t size = PyList_Size($input); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem($input, i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + PyObject *o2 = PyList_GetItem(o, j); + if (PyInt_Check(o2)) { + (*out)[i][j] = static_cast(PyInt_AsLong(o2)); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + $1 = out; +} + +%typemap(in) const std::unordered_map & { + std::unordered_map *out = nullptr; + if (PyDict_Check($input)) { + PyObject *key, *value; + Py_ssize_t pos = 0; + out = new std::unordered_map; + while (PyDict_Next($input, &pos, &key, &value)) { + const PyInputString key_ustring(key); + const PyInputString value_ustring(value); + if (key_ustring.IsAvalable() && value_ustring.IsAvalable()) { + out->emplace(std::string(key_ustring.data(), key_ustring.size()), + std::string(value_ustring.data(), value_ustring.size())); + } else { + PyErr_SetString(PyExc_TypeError, "map must contain strings."); + SWIG_fail; + } + resultobj = key_ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a dictionary"); + SWIG_fail; + } + $1 = out; +} + +%typemap(out) std::vector, float>> { + PyObject *input_type = resultobj; + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyObject *obj = PyList_New($1[i].first.size()); + for (size_t j = 0; j < $1[i].first.size(); ++j) { + PyList_SET_ITEM(obj, j, MakePyOutputString($1[i].first[j], input_type)); + } + PyList_SET_ITEM($result, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast($1[i].second)))); + } +} + +%typemap(out) std::vector, float>> { + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyObject *obj = PyList_New($1[i].first.size()); + for (size_t j = 0; j < $1[i].first.size(); ++j) { + PyList_SET_ITEM(obj, j, PyInt_FromLong(static_cast($1[i].first[j]))); + } + PyList_SET_ITEM($result, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast($1[i].second)))); + } +} + +%typemap(out) std::vector { + $result = PyList_New($1.size()); + for (size_t i = 0; i < $1.size(); ++i) { + PyObject *obj = SWIG_NewPointerObj(new sentencepiece::ImmutableSentencePieceText($1.at(i)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0); + PyList_SET_ITEM($result, i, obj); + } +} + +// Types for normalized string and offset +%typemap(out) std::pair> { + PyObject *input_type = resultobj; + if (PyInputString::IsUnicode(input_type)) { + sentencepiece::ConvertToUnicodeAlignment(arg2, $1.first, &$1.second); + } + PyObject *obj = PyList_New($1.second.size()); + for (size_t i = 0; i < $1.second.size(); ++i) { + PyList_SET_ITEM(obj, i, PyInt_FromLong(static_cast($1.second[i]))); + } + $result = PyTuple_Pack(2, MakePyOutputString($1.first, input_type), obj); +} + +%typemap(in) sentencepiece::SentenceIterator * { + sentencepiece::SentenceIterator *out = nullptr; + if (PyIter_Check($input)) { + out = new PySentenceIterator($input); + } else { + PyErr_SetString(PyExc_TypeError, "not a iterator"); + SWIG_fail; + } + $1 = out; +} + +%typemap(freearg) const std::string& { + delete $1; +} + +%typemap(freearg) const std::vector& { + delete $1; +} + +%typemap(freearg) const std::vector& { + delete $1; +} + +%typemap(freearg) const std::vector>& { + delete $1; +} + +%typemap(freearg) const std::vector& { + delete $1; +} + +%typemap(freearg) const std::vector& { + delete $1; +} + +%typemap(freearg) const std::vector>& { + delete $1; +} + +%typemap(freearg) const std::unordered_map & { + delete $1; +} + +%typemap(freearg) sentencepiece::SentenceIterator * { + delete $1; +} + +%typemap(freearg) sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece { + delete $1; +} + +%typemap(freearg) sentencepiece::ImmutableSentencePieceText { + delete $1; +} + +%typemap(freearg) sentencepiece::ImmutableNBestSentencePieceText { + delete $1; +} + +%include +%include + +%pythoncode %{ + +import re +import csv +import sys +import os +import importlib.resources +from io import StringIO +from io import BytesIO + + +def _add_snake_case(classname): + """Added snake_cased method from CammelCased method.""" + + snake_map = {} + for k, v in classname.__dict__.items(): + if re.match(r'^[A-Z]+', k): + snake = re.sub(r'(?= v.piece_size()): + raise IndexError('piece id is out of range.') + return func(v, n) + + def _batched_func(self, arg): + if type(arg) is list: + return [_func(self, n) for n in arg] + else: + return _func(self, arg) + + setattr(classname, name, _batched_func) + + +_sentencepiece_processor_init_native = SentencePieceProcessor.__init__ +_sentencepiece_normalizer_init_native = SentencePieceNormalizer.__init__ +setattr(SentencePieceProcessor, '__init__', SentencePieceProcessor.Init) +setattr(SentencePieceNormalizer, '__init__', SentencePieceNormalizer.Init) + +SentencePieceProcessor.Tokenize = SentencePieceProcessor.Encode +SentencePieceProcessor.Detokenize = SentencePieceProcessor.Decode + +for m in [ + 'PieceToId', 'IdToPiece', 'GetScore', 'IsUnknown', 'IsControl', 'IsUnused', + 'IsByte' +]: + _batchnize(SentencePieceProcessor, m) + +_add_snake_case(SentencePieceProcessor) +_add_snake_case(SentencePieceTrainer) +_add_snake_case(SentencePieceNormalizer) +set_random_generator_seed = SetRandomGeneratorSeed +set_min_log_level = SetMinLogLevel + +from ._version import __version__ + +SetDataDir(os.path.join(str(importlib.resources.files('sentencepiece')), 'package_data')) + +class _LogStream(object): + def __init__(self, ostream=None): + self.ostream = ostream + if self.ostream is not None: + self.orig_stream_fileno = sys.stderr.fileno() + + def __enter__(self): + if self.ostream is not None: + self.orig_stream_dup = os.dup(self.orig_stream_fileno) + os.dup2(self.ostream.fileno(), self.orig_stream_fileno) + + def __exit__(self, type, value, traceback): + if self.ostream is not None: + os.close(self.orig_stream_fileno) + os.dup2(self.orig_stream_dup, self.orig_stream_fileno) + os.close(self.orig_stream_dup) + self.ostream.close() +%} diff --git a/lib/python3.12/site-packages/sentencepiece/sentencepiece_model_pb2.py b/lib/python3.12/site-packages/sentencepiece/sentencepiece_model_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..b07107d69de178e107544a29bb4d0280b5482241 --- /dev/null +++ b/lib/python3.12/site-packages/sentencepiece/sentencepiece_model_pb2.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: sentencepiece_model.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19sentencepiece_model.proto\x12\rsentencepiece\"\x80\x0c\n\x0bTrainerSpec\x12\r\n\x05input\x18\x01 \x03(\t\x12\x14\n\x0cinput_format\x18\x07 \x01(\t\x12\x14\n\x0cmodel_prefix\x18\x02 \x01(\t\x12\x41\n\nmodel_type\x18\x03 \x01(\x0e\x32$.sentencepiece.TrainerSpec.ModelType:\x07UNIGRAM\x12\x18\n\nvocab_size\x18\x04 \x01(\x05:\x04\x38\x30\x30\x30\x12\x17\n\x0f\x61\x63\x63\x65pt_language\x18\x05 \x03(\t\x12 \n\x15self_test_sample_size\x18\x06 \x01(\x05:\x01\x30\x12*\n\x1b\x65nable_differential_privacy\x18\x32 \x01(\x08:\x05\x66\x61lse\x12+\n differential_privacy_noise_level\x18\x33 \x01(\x02:\x01\x30\x12\x32\n\'differential_privacy_clipping_threshold\x18\x34 \x01(\x04:\x01\x30\x12\"\n\x12\x63haracter_coverage\x18\n \x01(\x02:\x06\x30.9995\x12\x1e\n\x13input_sentence_size\x18\x0b \x01(\x04:\x01\x30\x12$\n\x16shuffle_input_sentence\x18\x13 \x01(\x08:\x04true\x12 \n\x14mining_sentence_size\x18\x0c \x01(\x05\x42\x02\x18\x01\x12\"\n\x16training_sentence_size\x18\r \x01(\x05\x42\x02\x18\x01\x12(\n\x17seed_sentencepiece_size\x18\x0e \x01(\x05:\x07\x31\x30\x30\x30\x30\x30\x30\x12\x1e\n\x10shrinking_factor\x18\x0f \x01(\x02:\x04\x30.75\x12!\n\x13max_sentence_length\x18\x12 \x01(\x05:\x04\x34\x31\x39\x32\x12\x17\n\x0bnum_threads\x18\x10 \x01(\x05:\x02\x31\x36\x12\x1d\n\x12num_sub_iterations\x18\x11 \x01(\x05:\x01\x32\x12$\n\x18max_sentencepiece_length\x18\x14 \x01(\x05:\x02\x31\x36\x12%\n\x17split_by_unicode_script\x18\x15 \x01(\x08:\x04true\x12\x1d\n\x0fsplit_by_number\x18\x17 \x01(\x08:\x04true\x12!\n\x13split_by_whitespace\x18\x16 \x01(\x08:\x04true\x12)\n\x1atreat_whitespace_as_suffix\x18\x18 \x01(\x08:\x05\x66\x61lse\x12+\n\x1c\x61llow_whitespace_only_pieces\x18\x1a \x01(\x08:\x05\x66\x61lse\x12\x1b\n\x0csplit_digits\x18\x19 \x01(\x08:\x05\x66\x61lse\x12#\n\x19pretokenization_delimiter\x18\x35 \x01(\t:\x00\x12\x17\n\x0f\x63ontrol_symbols\x18\x1e \x03(\t\x12\x1c\n\x14user_defined_symbols\x18\x1f \x03(\t\x12\x16\n\x0erequired_chars\x18$ \x01(\t\x12\x1c\n\rbyte_fallback\x18# \x01(\x08:\x05\x66\x61lse\x12+\n\x1dvocabulary_output_piece_score\x18 \x01(\x08:\x04true\x12\x1e\n\x10hard_vocab_limit\x18! \x01(\x08:\x04true\x12\x1c\n\ruse_all_vocab\x18\" \x01(\x08:\x05\x66\x61lse\x12\x11\n\x06unk_id\x18( \x01(\x05:\x01\x30\x12\x11\n\x06\x62os_id\x18) \x01(\x05:\x01\x31\x12\x11\n\x06\x65os_id\x18* \x01(\x05:\x01\x32\x12\x12\n\x06pad_id\x18+ \x01(\x05:\x02-1\x12\x18\n\tunk_piece\x18- \x01(\t:\x05\x12\x16\n\tbos_piece\x18. \x01(\t:\x03\x12\x17\n\teos_piece\x18/ \x01(\t:\x04\x12\x18\n\tpad_piece\x18\x30 \x01(\t:\x05\x12\x1a\n\x0bunk_surface\x18, \x01(\t:\x05 \xe2\x81\x87 \x12+\n\x1ctrain_extremely_large_corpus\x18\x31 \x01(\x08:\x05\x66\x61lse\"5\n\tModelType\x12\x0b\n\x07UNIGRAM\x10\x01\x12\x07\n\x03\x42PE\x10\x02\x12\x08\n\x04WORD\x10\x03\x12\x08\n\x04\x43HAR\x10\x04*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xd1\x01\n\x0eNormalizerSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x1c\n\x14precompiled_charsmap\x18\x02 \x01(\x0c\x12\x1e\n\x10\x61\x64\x64_dummy_prefix\x18\x03 \x01(\x08:\x04true\x12&\n\x18remove_extra_whitespaces\x18\x04 \x01(\x08:\x04true\x12 \n\x12\x65scape_whitespaces\x18\x05 \x01(\x08:\x04true\x12\x1e\n\x16normalization_rule_tsv\x18\x06 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"y\n\x0cSelfTestData\x12\x33\n\x07samples\x18\x01 \x03(\x0b\x32\".sentencepiece.SelfTestData.Sample\x1a)\n\x06Sample\x12\r\n\x05input\x18\x01 \x01(\t\x12\x10\n\x08\x65xpected\x18\x02 \x01(\t*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"\xfe\x03\n\nModelProto\x12\x37\n\x06pieces\x18\x01 \x03(\x0b\x32\'.sentencepiece.ModelProto.SentencePiece\x12\x30\n\x0ctrainer_spec\x18\x02 \x01(\x0b\x32\x1a.sentencepiece.TrainerSpec\x12\x36\n\x0fnormalizer_spec\x18\x03 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x12\x33\n\x0eself_test_data\x18\x04 \x01(\x0b\x32\x1b.sentencepiece.SelfTestData\x12\x38\n\x11\x64\x65normalizer_spec\x18\x05 \x01(\x0b\x32\x1d.sentencepiece.NormalizerSpec\x1a\xd2\x01\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\r\n\x05score\x18\x02 \x01(\x02\x12\x42\n\x04type\x18\x03 \x01(\x0e\x32,.sentencepiece.ModelProto.SentencePiece.Type:\x06NORMAL\"T\n\x04Type\x12\n\n\x06NORMAL\x10\x01\x12\x0b\n\x07UNKNOWN\x10\x02\x12\x0b\n\x07\x43ONTROL\x10\x03\x12\x10\n\x0cUSER_DEFINED\x10\x04\x12\x08\n\x04\x42YTE\x10\x06\x12\n\n\x06UNUSED\x10\x05*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\x42\x02H\x03') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sentencepiece_model_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'H\003' + _TRAINERSPEC.fields_by_name['mining_sentence_size']._options = None + _TRAINERSPEC.fields_by_name['mining_sentence_size']._serialized_options = b'\030\001' + _TRAINERSPEC.fields_by_name['training_sentence_size']._options = None + _TRAINERSPEC.fields_by_name['training_sentence_size']._serialized_options = b'\030\001' + _TRAINERSPEC._serialized_start=45 + _TRAINERSPEC._serialized_end=1581 + _TRAINERSPEC_MODELTYPE._serialized_start=1517 + _TRAINERSPEC_MODELTYPE._serialized_end=1570 + _NORMALIZERSPEC._serialized_start=1584 + _NORMALIZERSPEC._serialized_end=1793 + _SELFTESTDATA._serialized_start=1795 + _SELFTESTDATA._serialized_end=1916 + _SELFTESTDATA_SAMPLE._serialized_start=1864 + _SELFTESTDATA_SAMPLE._serialized_end=1905 + _MODELPROTO._serialized_start=1919 + _MODELPROTO._serialized_end=2429 + _MODELPROTO_SENTENCEPIECE._serialized_start=2208 + _MODELPROTO_SENTENCEPIECE._serialized_end=2418 + _MODELPROTO_SENTENCEPIECE_TYPE._serialized_start=2323 + _MODELPROTO_SENTENCEPIECE_TYPE._serialized_end=2407 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/sentencepiece/sentencepiece_pb2.py b/lib/python3.12/site-packages/sentencepiece/sentencepiece_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..840cfd2143a47d8a972008bd946a8dd271aa0a0f --- /dev/null +++ b/lib/python3.12/site-packages/sentencepiece/sentencepiece_pb2.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: sentencepiece.proto +"""Generated protocol buffer code.""" +from google.protobuf.internal import builder as _builder +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x13sentencepiece.proto\x12\rsentencepiece\"\xdf\x01\n\x11SentencePieceText\x12\x0c\n\x04text\x18\x01 \x01(\t\x12>\n\x06pieces\x18\x02 \x03(\x0b\x32..sentencepiece.SentencePieceText.SentencePiece\x12\r\n\x05score\x18\x03 \x01(\x02\x1a\x62\n\rSentencePiece\x12\r\n\x05piece\x18\x01 \x01(\t\x12\n\n\x02id\x18\x02 \x01(\r\x12\x0f\n\x07surface\x18\x03 \x01(\t\x12\r\n\x05\x62\x65gin\x18\x04 \x01(\r\x12\x0b\n\x03\x65nd\x18\x05 \x01(\r*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02*\t\x08\xc8\x01\x10\x80\x80\x80\x80\x02\"J\n\x16NBestSentencePieceText\x12\x30\n\x06nbests\x18\x01 \x03(\x0b\x32 .sentencepiece.SentencePieceTextB\x02H\x03') + +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'sentencepiece_pb2', globals()) +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + DESCRIPTOR._serialized_options = b'H\003' + _SENTENCEPIECETEXT._serialized_start=39 + _SENTENCEPIECETEXT._serialized_end=262 + _SENTENCEPIECETEXT_SENTENCEPIECE._serialized_start=153 + _SENTENCEPIECETEXT_SENTENCEPIECE._serialized_end=251 + _NBESTSENTENCEPIECETEXT._serialized_start=264 + _NBESTSENTENCEPIECETEXT._serialized_end=338 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/sentencepiece/sentencepiece_wrap.cxx b/lib/python3.12/site-packages/sentencepiece/sentencepiece_wrap.cxx new file mode 100644 index 0000000000000000000000000000000000000000..e6c3d659469e6cbe9c7bab2d8349fa289cd758b1 --- /dev/null +++ b/lib/python3.12/site-packages/sentencepiece/sentencepiece_wrap.cxx @@ -0,0 +1,10628 @@ +/* ---------------------------------------------------------------------------- + * This file was automatically generated by SWIG (https://www.swig.org). + * Version 4.3.0 + * + * Do not make changes to this file unless you know what you are doing - modify + * the SWIG interface file instead. + * ----------------------------------------------------------------------------- */ + + +#define SWIG_VERSION 0x040300 +#define SWIGPYTHON +#define SWIG_PYTHON_DIRECTOR_NO_VTABLE + +#define SWIG_name "_sentencepiece" +/* ----------------------------------------------------------------------------- + * This section contains generic SWIG labels for method/variable + * declarations/attributes, and other compiler dependent labels. + * ----------------------------------------------------------------------------- */ + +/* template workaround for compilers that cannot correctly implement the C++ standard */ +#ifndef SWIGTEMPLATEDISAMBIGUATOR +# if defined(__SUNPRO_CC) && (__SUNPRO_CC <= 0x560) +# define SWIGTEMPLATEDISAMBIGUATOR template +# elif defined(__HP_aCC) +/* Needed even with `aCC -AA' when `aCC -V' reports HP ANSI C++ B3910B A.03.55 */ +/* If we find a maximum version that requires this, the test would be __HP_aCC <= 35500 for A.03.55 */ +# define SWIGTEMPLATEDISAMBIGUATOR template +# else +# define SWIGTEMPLATEDISAMBIGUATOR +# endif +#endif + +/* inline attribute */ +#ifndef SWIGINLINE +# if defined(__cplusplus) || (defined(__GNUC__) && !defined(__STRICT_ANSI__)) +# define SWIGINLINE inline +# else +# define SWIGINLINE +# endif +#endif + +/* attribute recognised by some compilers to avoid 'unused' warnings */ +#ifndef SWIGUNUSED +# if defined(__GNUC__) +# if !(defined(__cplusplus)) || (__GNUC__ > 3 || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4)) +# define SWIGUNUSED __attribute__ ((__unused__)) +# else +# define SWIGUNUSED +# endif +# elif defined(__ICC) +# define SWIGUNUSED __attribute__ ((__unused__)) +# else +# define SWIGUNUSED +# endif +#endif + +#ifndef SWIG_MSC_UNSUPPRESS_4505 +# if defined(_MSC_VER) +# pragma warning(disable : 4505) /* unreferenced local function has been removed */ +# endif +#endif + +#ifndef SWIGUNUSEDPARM +# ifdef __cplusplus +# define SWIGUNUSEDPARM(p) +# else +# define SWIGUNUSEDPARM(p) p SWIGUNUSED +# endif +#endif + +/* internal SWIG method */ +#ifndef SWIGINTERN +# define SWIGINTERN static SWIGUNUSED +#endif + +/* internal inline SWIG method */ +#ifndef SWIGINTERNINLINE +# define SWIGINTERNINLINE SWIGINTERN SWIGINLINE +#endif + +/* exporting methods */ +#if defined(__GNUC__) +# if (__GNUC__ >= 4) || (__GNUC__ == 3 && __GNUC_MINOR__ >= 4) +# ifndef GCC_HASCLASSVISIBILITY +# define GCC_HASCLASSVISIBILITY +# endif +# endif +#endif + +#ifndef SWIGEXPORT +# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) +# if defined(STATIC_LINKED) +# define SWIGEXPORT +# else +# define SWIGEXPORT __declspec(dllexport) +# endif +# else +# if defined(__GNUC__) && defined(GCC_HASCLASSVISIBILITY) +# define SWIGEXPORT __attribute__ ((visibility("default"))) +# else +# define SWIGEXPORT +# endif +# endif +#endif + +/* calling conventions for Windows */ +#ifndef SWIGSTDCALL +# if defined(_WIN32) || defined(__WIN32__) || defined(__CYGWIN__) +# define SWIGSTDCALL __stdcall +# else +# define SWIGSTDCALL +# endif +#endif + +/* Deal with Microsoft's attempt at deprecating C standard runtime functions */ +#if !defined(SWIG_NO_CRT_SECURE_NO_DEPRECATE) && defined(_MSC_VER) && !defined(_CRT_SECURE_NO_DEPRECATE) +# define _CRT_SECURE_NO_DEPRECATE +#endif + +/* Deal with Microsoft's attempt at deprecating methods in the standard C++ library */ +#if !defined(SWIG_NO_SCL_SECURE_NO_DEPRECATE) && defined(_MSC_VER) && !defined(_SCL_SECURE_NO_DEPRECATE) +# define _SCL_SECURE_NO_DEPRECATE +#endif + +/* Deal with Apple's deprecated 'AssertMacros.h' from Carbon-framework */ +#if defined(__APPLE__) && !defined(__ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES) +# define __ASSERT_MACROS_DEFINE_VERSIONS_WITHOUT_UNDERSCORES 0 +#endif + +/* Intel's compiler complains if a variable which was never initialised is + * cast to void, which is a common idiom which we use to indicate that we + * are aware a variable isn't used. So we just silence that warning. + * See: https://github.com/swig/swig/issues/192 for more discussion. + */ +#ifdef __INTEL_COMPILER +# pragma warning disable 592 +#endif + +#if defined(__cplusplus) && __cplusplus >=201103L +# define SWIG_NULLPTR nullptr +#else +# define SWIG_NULLPTR NULL +#endif + +/* ----------------------------------------------------------------------------- + * swigcompat.swg + * + * Macros to provide support compatibility with older C and C++ standards. + * + * Note that SWIG expects __cplusplus to be defined to the appropriate C++ standard. + * MSVC users are urged to check and examine the /Zc:__cplusplus compiler option. + * See https://learn.microsoft.com/en-us/cpp/build/reference/zc-cplusplus. + * ----------------------------------------------------------------------------- */ + +/* C99 and C++11 should provide snprintf, but define SWIG_NO_SNPRINTF + * if you're missing it. + */ +#if ((defined __STDC_VERSION__ && __STDC_VERSION__ >= 199901L) || \ + (defined __cplusplus && __cplusplus >= 201103L) || \ + defined SWIG_HAVE_SNPRINTF) && \ + !defined SWIG_NO_SNPRINTF +# define SWIG_snprintf(O,S,F,A) snprintf(O,S,F,A) +# define SWIG_snprintf2(O,S,F,A,B) snprintf(O,S,F,A,B) +#else +/* Fallback versions ignore the buffer size, but most of our uses either have a + * fixed maximum possible size or dynamically allocate a buffer that's large + * enough. + */ +# define SWIG_snprintf(O,S,F,A) sprintf(O,F,A) +# define SWIG_snprintf2(O,S,F,A,B) sprintf(O,F,A,B) +#endif + + +#if defined(__GNUC__) && defined(_WIN32) && !defined(SWIG_PYTHON_NO_HYPOT_WORKAROUND) +/* Workaround for '::hypot' has not been declared', see https://bugs.python.org/issue11566 */ +# include +#endif + +#if !defined(PY_SSIZE_T_CLEAN) && !defined(SWIG_NO_PY_SSIZE_T_CLEAN) +#define PY_SSIZE_T_CLEAN +#endif + +#if __GNUC__ >= 7 +#pragma GCC diagnostic push +#if defined(__cplusplus) && __cplusplus >=201703L +#pragma GCC diagnostic ignored "-Wregister" /* For python-2.7 headers that use register */ +#endif +#endif + +#if defined(_DEBUG) && defined(SWIG_PYTHON_INTERPRETER_NO_DEBUG) +/* Use debug wrappers with the Python release dll */ + +#if defined(_MSC_VER) && _MSC_VER >= 1929 +/* Workaround compilation errors when redefining _DEBUG in MSVC 2019 version 16.10 and later + * See https://github.com/swig/swig/issues/2090 */ +# include +#endif + +# undef _DEBUG +# include +# define _DEBUG 1 +#else +# include +#endif + +#if !defined(SWIGPYTHON_BUILTIN) && PY_VERSION_HEX >= 0x03030000 +# define SWIG_HEAPTYPES + +/* Note: Currently this won't activate - it is in place ready for when the + * SWIGPYTHON_BUILTIN condition above gets removed. */ +# if PY_VERSION_HEX < 0x030c0000 && defined(SWIGPYTHON_BUILTIN) +# include +# define Py_READONLY READONLY +# define Py_T_PYSSIZET T_PYSSIZET +# endif +#endif + +#if __GNUC__ >= 7 +#pragma GCC diagnostic pop +#endif + +#include +#include + +/* ----------------------------------------------------------------------------- + * swigrun.swg + * + * This file contains generic C API SWIG runtime support for pointer + * type checking. + * ----------------------------------------------------------------------------- */ + +/* This should only be incremented when either the layout of swig_type_info changes, + or for whatever reason, the runtime changes incompatibly */ +#define SWIG_RUNTIME_VERSION "4" + +/* define SWIG_TYPE_TABLE_NAME as "SWIG_TYPE_TABLE" */ +#ifdef SWIG_TYPE_TABLE +# define SWIG_QUOTE_STRING(x) #x +# define SWIG_EXPAND_AND_QUOTE_STRING(x) SWIG_QUOTE_STRING(x) +# define SWIG_TYPE_TABLE_NAME SWIG_EXPAND_AND_QUOTE_STRING(SWIG_TYPE_TABLE) +#else +# define SWIG_TYPE_TABLE_NAME +#endif + +/* + You can use the SWIGRUNTIME and SWIGRUNTIMEINLINE macros for + creating a static or dynamic library from the SWIG runtime code. + In 99.9% of the cases, SWIG just needs to declare them as 'static'. + + But only do this if strictly necessary, ie, if you have problems + with your compiler or suchlike. +*/ + +#ifndef SWIGRUNTIME +# define SWIGRUNTIME SWIGINTERN +#endif + +#ifndef SWIGRUNTIMEINLINE +# define SWIGRUNTIMEINLINE SWIGRUNTIME SWIGINLINE +#endif + +/* Generic buffer size */ +#ifndef SWIG_BUFFER_SIZE +# define SWIG_BUFFER_SIZE 1024 +#endif + +/* Flags for pointer conversions */ +#define SWIG_POINTER_DISOWN 0x1 +#define SWIG_CAST_NEW_MEMORY 0x2 +#define SWIG_POINTER_NO_NULL 0x4 +#define SWIG_POINTER_CLEAR 0x8 +#define SWIG_POINTER_RELEASE (SWIG_POINTER_CLEAR | SWIG_POINTER_DISOWN) + +/* Flags for new pointer objects */ +#define SWIG_POINTER_OWN 0x1 + + +/* + Flags/methods for returning states. + + The SWIG conversion methods, as ConvertPtr, return an integer + that tells if the conversion was successful or not. And if not, + an error code can be returned (see swigerrors.swg for the codes). + + Use the following macros/flags to set or process the returning + states. + + In old versions of SWIG, code such as the following was usually written: + + if (SWIG_ConvertPtr(obj,vptr,ty.flags) != -1) { + // success code + } else { + //fail code + } + + Now you can be more explicit: + + int res = SWIG_ConvertPtr(obj,vptr,ty.flags); + if (SWIG_IsOK(res)) { + // success code + } else { + // fail code + } + + which is the same really, but now you can also do + + Type *ptr; + int res = SWIG_ConvertPtr(obj,(void **)(&ptr),ty.flags); + if (SWIG_IsOK(res)) { + // success code + if (SWIG_IsNewObj(res) { + ... + delete *ptr; + } else { + ... + } + } else { + // fail code + } + + I.e., now SWIG_ConvertPtr can return new objects and you can + identify the case and take care of the deallocation. Of course that + also requires SWIG_ConvertPtr to return new result values, such as + + int SWIG_ConvertPtr(obj, ptr,...) { + if () { + if () { + *ptr = ; + return SWIG_NEWOBJ; + } else { + *ptr = ; + return SWIG_OLDOBJ; + } + } else { + return SWIG_BADOBJ; + } + } + + Of course, returning the plain '0(success)/-1(fail)' still works, but you can be + more explicit by returning SWIG_BADOBJ, SWIG_ERROR or any of the + SWIG errors code. + + Finally, if the SWIG_CASTRANK_MODE is enabled, the result code + allows returning the 'cast rank', for example, if you have this + + int food(double) + int fooi(int); + + and you call + + food(1) // cast rank '1' (1 -> 1.0) + fooi(1) // cast rank '0' + + just use the SWIG_AddCast()/SWIG_CheckState() +*/ + +#define SWIG_OK (0) +/* Runtime errors are < 0 */ +#define SWIG_ERROR (-1) +/* Errors in range -1 to -99 are in swigerrors.swg (errors for all languages including those not using the runtime) */ +/* Errors in range -100 to -199 are language specific errors defined in *errors.swg */ +/* Errors < -200 are generic runtime specific errors */ +#define SWIG_ERROR_RELEASE_NOT_OWNED (-200) + +#define SWIG_IsOK(r) (r >= 0) +#define SWIG_ArgError(r) ((r != SWIG_ERROR) ? r : SWIG_TypeError) + +/* The CastRankLimit says how many bits are used for the cast rank */ +#define SWIG_CASTRANKLIMIT (1 << 8) +/* The NewMask denotes the object was created (using new/malloc) */ +#define SWIG_NEWOBJMASK (SWIG_CASTRANKLIMIT << 1) +/* The TmpMask is for in/out typemaps that use temporary objects */ +#define SWIG_TMPOBJMASK (SWIG_NEWOBJMASK << 1) +/* Simple returning values */ +#define SWIG_BADOBJ (SWIG_ERROR) +#define SWIG_OLDOBJ (SWIG_OK) +#define SWIG_NEWOBJ (SWIG_OK | SWIG_NEWOBJMASK) +#define SWIG_TMPOBJ (SWIG_OK | SWIG_TMPOBJMASK) +/* Check, add and del object mask methods */ +#define SWIG_AddNewMask(r) (SWIG_IsOK(r) ? (r | SWIG_NEWOBJMASK) : r) +#define SWIG_DelNewMask(r) (SWIG_IsOK(r) ? (r & ~SWIG_NEWOBJMASK) : r) +#define SWIG_IsNewObj(r) (SWIG_IsOK(r) && (r & SWIG_NEWOBJMASK)) +#define SWIG_AddTmpMask(r) (SWIG_IsOK(r) ? (r | SWIG_TMPOBJMASK) : r) +#define SWIG_DelTmpMask(r) (SWIG_IsOK(r) ? (r & ~SWIG_TMPOBJMASK) : r) +#define SWIG_IsTmpObj(r) (SWIG_IsOK(r) && (r & SWIG_TMPOBJMASK)) + +/* Cast-Rank Mode */ +#if defined(SWIG_CASTRANK_MODE) +# ifndef SWIG_TypeRank +# define SWIG_TypeRank unsigned long +# endif +# ifndef SWIG_MAXCASTRANK /* Default cast allowed */ +# define SWIG_MAXCASTRANK (2) +# endif +# define SWIG_CASTRANKMASK ((SWIG_CASTRANKLIMIT) -1) +# define SWIG_CastRank(r) (r & SWIG_CASTRANKMASK) +SWIGINTERNINLINE int SWIG_AddCast(int r) { + return SWIG_IsOK(r) ? ((SWIG_CastRank(r) < SWIG_MAXCASTRANK) ? (r + 1) : SWIG_ERROR) : r; +} +SWIGINTERNINLINE int SWIG_CheckState(int r) { + return SWIG_IsOK(r) ? SWIG_CastRank(r) + 1 : 0; +} +#else /* no cast-rank mode */ +# define SWIG_AddCast(r) (r) +# define SWIG_CheckState(r) (SWIG_IsOK(r) ? 1 : 0) +#endif + + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +typedef void *(*swig_converter_func)(void *, int *); +typedef struct swig_type_info *(*swig_dycast_func)(void **); + +/* Structure to store information on one type */ +typedef struct swig_type_info { + const char *name; /* mangled name of this type */ + const char *str; /* human readable name of this type */ + swig_dycast_func dcast; /* dynamic cast function down a hierarchy */ + struct swig_cast_info *cast; /* linked list of types that can cast into this type */ + void *clientdata; /* language specific type data */ + int owndata; /* flag if the structure owns the clientdata */ +} swig_type_info; + +/* Structure to store a type and conversion function used for casting */ +typedef struct swig_cast_info { + swig_type_info *type; /* pointer to type that is equivalent to this type */ + swig_converter_func converter; /* function to cast the void pointers */ + struct swig_cast_info *next; /* pointer to next cast in linked list */ + struct swig_cast_info *prev; /* pointer to the previous cast */ +} swig_cast_info; + +/* Structure used to store module information + * Each module generates one structure like this, and the runtime collects + * all of these structures and stores them in a circularly linked list.*/ +typedef struct swig_module_info { + swig_type_info **types; /* Array of pointers to swig_type_info structures that are in this module */ + size_t size; /* Number of types in this module */ + struct swig_module_info *next; /* Pointer to next element in circularly linked list */ + swig_type_info **type_initial; /* Array of initially generated type structures */ + swig_cast_info **cast_initial; /* Array of initially generated casting structures */ + void *clientdata; /* Language specific module data */ +} swig_module_info; + +/* + Compare two type names skipping the space characters, therefore + "char*" == "char *" and "Class" == "Class", etc. + + Return 0 when the two name types are equivalent, as in + strncmp, but skipping ' '. +*/ +SWIGRUNTIME int +SWIG_TypeNameComp(const char *f1, const char *l1, + const char *f2, const char *l2) { + for (;(f1 != l1) && (f2 != l2); ++f1, ++f2) { + while ((*f1 == ' ') && (f1 != l1)) ++f1; + while ((*f2 == ' ') && (f2 != l2)) ++f2; + if (*f1 != *f2) return (*f1 > *f2) ? 1 : -1; + } + return (int)((l1 - f1) - (l2 - f2)); +} + +/* + Check type equivalence in a name list like ||... + Return 0 if equal, -1 if nb < tb, 1 if nb > tb +*/ +SWIGRUNTIME int +SWIG_TypeCmp(const char *nb, const char *tb) { + int equiv = 1; + const char* te = tb + strlen(tb); + const char* ne = nb; + while (equiv != 0 && *ne) { + for (nb = ne; *ne; ++ne) { + if (*ne == '|') break; + } + equiv = SWIG_TypeNameComp(nb, ne, tb, te); + if (*ne) ++ne; + } + return equiv; +} + +/* + Check type equivalence in a name list like ||... + Return 0 if not equal, 1 if equal +*/ +SWIGRUNTIME int +SWIG_TypeEquiv(const char *nb, const char *tb) { + return SWIG_TypeCmp(nb, tb) == 0 ? 1 : 0; +} + +/* + Check the typename +*/ +SWIGRUNTIME swig_cast_info * +SWIG_TypeCheck(const char *c, swig_type_info *ty) { + if (ty) { + swig_cast_info *iter = ty->cast; + while (iter) { + if (strcmp(iter->type->name, c) == 0) { + if (iter == ty->cast) + return iter; + /* Move iter to the top of the linked list */ + iter->prev->next = iter->next; + if (iter->next) + iter->next->prev = iter->prev; + iter->next = ty->cast; + iter->prev = 0; + if (ty->cast) ty->cast->prev = iter; + ty->cast = iter; + return iter; + } + iter = iter->next; + } + } + return 0; +} + +/* + Identical to SWIG_TypeCheck, except strcmp is replaced with a pointer comparison +*/ +SWIGRUNTIME swig_cast_info * +SWIG_TypeCheckStruct(const swig_type_info *from, swig_type_info *ty) { + if (ty) { + swig_cast_info *iter = ty->cast; + while (iter) { + if (iter->type == from) { + if (iter == ty->cast) + return iter; + /* Move iter to the top of the linked list */ + iter->prev->next = iter->next; + if (iter->next) + iter->next->prev = iter->prev; + iter->next = ty->cast; + iter->prev = 0; + if (ty->cast) ty->cast->prev = iter; + ty->cast = iter; + return iter; + } + iter = iter->next; + } + } + return 0; +} + +/* + Cast a pointer up an inheritance hierarchy +*/ +SWIGRUNTIMEINLINE void * +SWIG_TypeCast(swig_cast_info *ty, void *ptr, int *newmemory) { + return ((!ty) || (!ty->converter)) ? ptr : (*ty->converter)(ptr, newmemory); +} + +/* + Dynamic pointer casting. Down an inheritance hierarchy +*/ +SWIGRUNTIME swig_type_info * +SWIG_TypeDynamicCast(swig_type_info *ty, void **ptr) { + swig_type_info *lastty = ty; + if (!ty || !ty->dcast) return ty; + while (ty && (ty->dcast)) { + ty = (*ty->dcast)(ptr); + if (ty) lastty = ty; + } + return lastty; +} + +/* + Return the name associated with this type +*/ +SWIGRUNTIMEINLINE const char * +SWIG_TypeName(const swig_type_info *ty) { + return ty->name; +} + +/* + Return the pretty name associated with this type, + that is an unmangled type name in a form presentable to the user. +*/ +SWIGRUNTIME const char * +SWIG_TypePrettyName(const swig_type_info *type) { + /* The "str" field contains the equivalent pretty names of the + type, separated by vertical-bar characters. Choose the last + name. It should be the most specific; a fully resolved name + but not necessarily with default template parameters expanded. */ + if (!type) return NULL; + if (type->str != NULL) { + const char *last_name = type->str; + const char *s; + for (s = type->str; *s; s++) + if (*s == '|') last_name = s+1; + return last_name; + } + else + return type->name; +} + +/* + Set the clientdata field for a type +*/ +SWIGRUNTIME void +SWIG_TypeClientData(swig_type_info *ti, void *clientdata) { + swig_cast_info *cast = ti->cast; + /* if (ti->clientdata == clientdata) return; */ + ti->clientdata = clientdata; + + while (cast) { + if (!cast->converter) { + swig_type_info *tc = cast->type; + if (!tc->clientdata) { + SWIG_TypeClientData(tc, clientdata); + } + } + cast = cast->next; + } +} +SWIGRUNTIME void +SWIG_TypeNewClientData(swig_type_info *ti, void *clientdata) { + SWIG_TypeClientData(ti, clientdata); + ti->owndata = 1; +} + +/* + Search for a swig_type_info structure only by mangled name + Search is a O(log #types) + + We start searching at module start, and finish searching when start == end. + Note: if start == end at the beginning of the function, we go all the way around + the circular list. +*/ +SWIGRUNTIME swig_type_info * +SWIG_MangledTypeQueryModule(swig_module_info *start, + swig_module_info *end, + const char *name) { + swig_module_info *iter = start; + do { + if (iter->size) { + size_t l = 0; + size_t r = iter->size - 1; + do { + /* since l+r >= 0, we can (>> 1) instead (/ 2) */ + size_t i = (l + r) >> 1; + const char *iname = iter->types[i]->name; + if (iname) { + int compare = strcmp(name, iname); + if (compare == 0) { + return iter->types[i]; + } else if (compare < 0) { + if (i) { + r = i - 1; + } else { + break; + } + } else if (compare > 0) { + l = i + 1; + } + } else { + break; /* should never happen */ + } + } while (l <= r); + } + iter = iter->next; + } while (iter != end); + return 0; +} + +/* + Search for a swig_type_info structure for either a mangled name or a human readable name. + It first searches the mangled names of the types, which is a O(log #types) + If a type is not found it then searches the human readable names, which is O(#types). + + We start searching at module start, and finish searching when start == end. + Note: if start == end at the beginning of the function, we go all the way around + the circular list. +*/ +SWIGRUNTIME swig_type_info * +SWIG_TypeQueryModule(swig_module_info *start, + swig_module_info *end, + const char *name) { + /* STEP 1: Search the name field using binary search */ + swig_type_info *ret = SWIG_MangledTypeQueryModule(start, end, name); + if (ret) { + return ret; + } else { + /* STEP 2: If the type hasn't been found, do a complete search + of the str field (the human readable name) */ + swig_module_info *iter = start; + do { + size_t i = 0; + for (; i < iter->size; ++i) { + if (iter->types[i]->str && (SWIG_TypeEquiv(iter->types[i]->str, name))) + return iter->types[i]; + } + iter = iter->next; + } while (iter != end); + } + + /* neither found a match */ + return 0; +} + +/* + Pack binary data into a string +*/ +SWIGRUNTIME char * +SWIG_PackData(char *c, void *ptr, size_t sz) { + static const char hex[17] = "0123456789abcdef"; + const unsigned char *u = (unsigned char *) ptr; + const unsigned char *eu = u + sz; + for (; u != eu; ++u) { + unsigned char uu = *u; + *(c++) = hex[(uu & 0xf0) >> 4]; + *(c++) = hex[uu & 0xf]; + } + return c; +} + +/* + Unpack binary data from a string +*/ +SWIGRUNTIME const char * +SWIG_UnpackData(const char *c, void *ptr, size_t sz) { + unsigned char *u = (unsigned char *) ptr; + const unsigned char *eu = u + sz; + for (; u != eu; ++u) { + char d = *(c++); + unsigned char uu; + if ((d >= '0') && (d <= '9')) + uu = (unsigned char)((d - '0') << 4); + else if ((d >= 'a') && (d <= 'f')) + uu = (unsigned char)((d - ('a'-10)) << 4); + else + return (char *) 0; + d = *(c++); + if ((d >= '0') && (d <= '9')) + uu |= (unsigned char)(d - '0'); + else if ((d >= 'a') && (d <= 'f')) + uu |= (unsigned char)(d - ('a'-10)); + else + return (char *) 0; + *u = uu; + } + return c; +} + +/* + Pack 'void *' into a string buffer. +*/ +SWIGRUNTIME char * +SWIG_PackVoidPtr(char *buff, void *ptr, const char *name, size_t bsz) { + char *r = buff; + if ((2*sizeof(void *) + 2) > bsz) return 0; + *(r++) = '_'; + r = SWIG_PackData(r,&ptr,sizeof(void *)); + if (strlen(name) + 1 > (bsz - (r - buff))) return 0; + strcpy(r,name); + return buff; +} + +SWIGRUNTIME const char * +SWIG_UnpackVoidPtr(const char *c, void **ptr, const char *name) { + if (*c != '_') { + if (strcmp(c,"NULL") == 0) { + *ptr = (void *) 0; + return name; + } else { + return 0; + } + } + return SWIG_UnpackData(++c,ptr,sizeof(void *)); +} + +SWIGRUNTIME char * +SWIG_PackDataName(char *buff, void *ptr, size_t sz, const char *name, size_t bsz) { + char *r = buff; + size_t lname = (name ? strlen(name) : 0); + if ((2*sz + 2 + lname) > bsz) return 0; + *(r++) = '_'; + r = SWIG_PackData(r,ptr,sz); + if (lname) { + strncpy(r,name,lname+1); + } else { + *r = 0; + } + return buff; +} + +SWIGRUNTIME const char * +SWIG_UnpackDataName(const char *c, void *ptr, size_t sz, const char *name) { + if (*c != '_') { + if (strcmp(c,"NULL") == 0) { + memset(ptr,0,sz); + return name; + } else { + return 0; + } + } + return SWIG_UnpackData(++c,ptr,sz); +} + +#ifdef __cplusplus +} +#endif + +/* SWIG Errors applicable to all language modules, values are reserved from -1 to -99 */ +#define SWIG_UnknownError -1 +#define SWIG_IOError -2 +#define SWIG_RuntimeError -3 +#define SWIG_IndexError -4 +#define SWIG_TypeError -5 +#define SWIG_DivisionByZero -6 +#define SWIG_OverflowError -7 +#define SWIG_SyntaxError -8 +#define SWIG_ValueError -9 +#define SWIG_SystemError -10 +#define SWIG_AttributeError -11 +#define SWIG_MemoryError -12 +#define SWIG_NullReferenceError -13 + + +/* Compatibility macros for Python 3 */ +#if PY_VERSION_HEX >= 0x03000000 + +#define PyClass_Check(obj) PyObject_IsInstance(obj, (PyObject *)&PyType_Type) +#define PyInt_Check(x) PyLong_Check(x) +#define PyInt_AsLong(x) PyLong_AsLong(x) +#define PyInt_FromLong(x) PyLong_FromLong(x) +#define PyInt_FromSize_t(x) PyLong_FromSize_t(x) +#define PyString_Check(name) PyBytes_Check(name) +#define PyString_FromString(x) PyUnicode_FromString(x) +#define PyString_Format(fmt, args) PyUnicode_Format(fmt, args) +#define PyString_AsString(str) PyBytes_AsString(str) +#define PyString_Size(str) PyBytes_Size(str) +#define PyString_InternFromString(key) PyUnicode_InternFromString(key) +#define Py_TPFLAGS_HAVE_CLASS Py_TPFLAGS_BASETYPE +#define _PyLong_FromSsize_t(x) PyLong_FromSsize_t(x) + +#endif + +/* SWIG APIs for compatibility of both Python 2 & 3 */ + +#if PY_VERSION_HEX >= 0x03000000 +# define SWIG_Python_str_FromFormat PyUnicode_FromFormat +#else +# define SWIG_Python_str_FromFormat PyString_FromFormat +#endif + + +/* Wrapper around PyUnicode_AsUTF8AndSize - call Py_XDECREF on the returned pbytes when finished with the returned string */ +SWIGINTERN const char * +SWIG_PyUnicode_AsUTF8AndSize(PyObject *str, Py_ssize_t *psize, PyObject **pbytes) +{ +#if PY_VERSION_HEX >= 0x03030000 +# if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x030A0000 + *pbytes = NULL; + return PyUnicode_AsUTF8AndSize(str, psize); +# else + const char *chars; + *pbytes = PyUnicode_AsUTF8String(str); + chars = *pbytes ? PyBytes_AsString(*pbytes) : NULL; + if (chars && psize) + *psize = PyBytes_Size(*pbytes); + return chars; +# endif +#else + char *chars = NULL; + *pbytes = NULL; + PyString_AsStringAndSize(str, &chars, psize); + return chars; +#endif +} + +SWIGINTERN PyObject* +SWIG_Python_str_FromChar(const char *c) +{ +#if PY_VERSION_HEX >= 0x03000000 + return PyUnicode_FromString(c); +#else + return PyString_FromString(c); +#endif +} + +/* SWIGPY_USE_CAPSULE is no longer used within SWIG itself, but some user interface files check for it. */ +# define SWIGPY_USE_CAPSULE +#ifdef SWIGPYTHON_BUILTIN +# define SWIGPY_CAPSULE_ATTR_NAME "type_pointer_capsule_builtin" SWIG_TYPE_TABLE_NAME +#else +# define SWIGPY_CAPSULE_ATTR_NAME "type_pointer_capsule" SWIG_TYPE_TABLE_NAME +#endif +# define SWIGPY_CAPSULE_NAME ("swig_runtime_data" SWIG_RUNTIME_VERSION "." SWIGPY_CAPSULE_ATTR_NAME) + +#if PY_VERSION_HEX < 0x03020000 +#define PyDescr_TYPE(x) (((PyDescrObject *)(x))->d_type) +#define PyDescr_NAME(x) (((PyDescrObject *)(x))->d_name) +#define Py_hash_t long +#endif + +#ifdef Py_LIMITED_API +# define PyTuple_GET_ITEM PyTuple_GetItem +/* Note that PyTuple_SetItem() has different semantics from PyTuple_SET_ITEM as it decref's the original tuple item, so in general they cannot be used + interchangeably. However in SWIG-generated code PyTuple_SET_ITEM is only used with newly initialized tuples without any items and for them this does work. */ +# define PyTuple_SET_ITEM PyTuple_SetItem +# define PyTuple_GET_SIZE PyTuple_Size +# define PyCFunction_GET_FLAGS PyCFunction_GetFlags +# define PyCFunction_GET_FUNCTION PyCFunction_GetFunction +# define PyCFunction_GET_SELF PyCFunction_GetSelf +# define PyList_GET_ITEM PyList_GetItem +# define PyList_SET_ITEM PyList_SetItem +# define PySliceObject PyObject +#endif + +/* Increment and Decrement wrappers - for portability when using the stable abi and for performance otherwise */ +#ifdef Py_LIMITED_API +# define SWIG_Py_INCREF Py_IncRef +# define SWIG_Py_XINCREF Py_IncRef +# define SWIG_Py_DECREF Py_DecRef +# define SWIG_Py_XDECREF Py_DecRef +#else +# define SWIG_Py_INCREF Py_INCREF +# define SWIG_Py_XINCREF Py_XINCREF +# define SWIG_Py_DECREF Py_DECREF +# define SWIG_Py_XDECREF Py_XDECREF +#endif + +/* ----------------------------------------------------------------------------- + * error manipulation + * ----------------------------------------------------------------------------- */ + +SWIGRUNTIME PyObject* +SWIG_Python_ErrorType(int code) { + PyObject* type = 0; + switch(code) { + case SWIG_MemoryError: + type = PyExc_MemoryError; + break; + case SWIG_IOError: + type = PyExc_IOError; + break; + case SWIG_RuntimeError: + type = PyExc_RuntimeError; + break; + case SWIG_IndexError: + type = PyExc_IndexError; + break; + case SWIG_TypeError: + type = PyExc_TypeError; + break; + case SWIG_DivisionByZero: + type = PyExc_ZeroDivisionError; + break; + case SWIG_OverflowError: + type = PyExc_OverflowError; + break; + case SWIG_SyntaxError: + type = PyExc_SyntaxError; + break; + case SWIG_ValueError: + type = PyExc_ValueError; + break; + case SWIG_SystemError: + type = PyExc_SystemError; + break; + case SWIG_AttributeError: + type = PyExc_AttributeError; + break; + case SWIG_NullReferenceError: + type = PyExc_TypeError; + break; + default: + type = PyExc_RuntimeError; + } + return type; +} + + +SWIGRUNTIME void +SWIG_Python_AddErrorMsg(const char* mesg) +{ + PyObject *type = 0; + PyObject *value = 0; + PyObject *traceback = 0; + + if (PyErr_Occurred()) + PyErr_Fetch(&type, &value, &traceback); + if (value) { + PyObject *old_str = PyObject_Str(value); + PyObject *bytes = NULL; + const char *tmp = SWIG_PyUnicode_AsUTF8AndSize(old_str, NULL, &bytes); + PyErr_Clear(); + SWIG_Py_XINCREF(type); + if (tmp) + PyErr_Format(type, "%s %s", tmp, mesg); + else + PyErr_Format(type, "%s", mesg); + SWIG_Py_XDECREF(bytes); + SWIG_Py_DECREF(old_str); + SWIG_Py_DECREF(value); + } else { + PyErr_SetString(PyExc_RuntimeError, mesg); + } +} + +SWIGRUNTIME int +SWIG_Python_TypeErrorOccurred(PyObject *obj) +{ + PyObject *error; + if (obj) + return 0; + error = PyErr_Occurred(); + return error && PyErr_GivenExceptionMatches(error, PyExc_TypeError); +} + +SWIGRUNTIME void +SWIG_Python_RaiseOrModifyTypeError(const char *message) +{ + if (SWIG_Python_TypeErrorOccurred(NULL)) { + /* Use existing TypeError to preserve stacktrace and enhance with given message */ + PyObject *newvalue; + PyObject *type = NULL, *value = NULL, *traceback = NULL; + PyErr_Fetch(&type, &value, &traceback); +#if PY_VERSION_HEX >= 0x03000000 + newvalue = PyUnicode_FromFormat("%S\nAdditional information:\n%s", value, message); +#else + newvalue = PyString_FromFormat("%s\nAdditional information:\n%s", PyString_AsString(value), message); +#endif + if (newvalue) { + SWIG_Py_XDECREF(value); + PyErr_Restore(type, newvalue, traceback); + } else { + PyErr_Restore(type, value, traceback); + } + } else { + /* Raise TypeError using given message */ + PyErr_SetString(PyExc_TypeError, message); + } +} + +#if defined(SWIG_PYTHON_NO_THREADS) +# if defined(SWIG_PYTHON_THREADS) +# undef SWIG_PYTHON_THREADS +# endif +#endif +#if defined(SWIG_PYTHON_THREADS) /* Threading support is enabled */ +# if !defined(SWIG_PYTHON_USE_GIL) && !defined(SWIG_PYTHON_NO_USE_GIL) +# define SWIG_PYTHON_USE_GIL +# endif +# if defined(SWIG_PYTHON_USE_GIL) /* Use PyGILState threads calls */ +# if !defined(SWIG_PYTHON_INITIALIZE_THREADS) +# if PY_VERSION_HEX < 0x03070000 +# define SWIG_PYTHON_INITIALIZE_THREADS PyEval_InitThreads() +# else +# define SWIG_PYTHON_INITIALIZE_THREADS +# endif +# endif +# ifdef __cplusplus /* C++ code */ + class SWIG_Python_Thread_Block { + bool status; + PyGILState_STATE state; + public: + void end() { if (status) { PyGILState_Release(state); status = false;} } + SWIG_Python_Thread_Block() : status(true), state(PyGILState_Ensure()) {} + ~SWIG_Python_Thread_Block() { end(); } + }; + class SWIG_Python_Thread_Allow { + bool status; + PyThreadState *save; + public: + void end() { if (status) { status = false; PyEval_RestoreThread(save); }} + SWIG_Python_Thread_Allow() : status(true), save(PyEval_SaveThread()) {} + ~SWIG_Python_Thread_Allow() { end(); } + }; +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK SWIG_Python_Thread_Block _swig_thread_block +# define SWIG_PYTHON_THREAD_END_BLOCK _swig_thread_block.end() +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW SWIG_Python_Thread_Allow _swig_thread_allow +# define SWIG_PYTHON_THREAD_END_ALLOW _swig_thread_allow.end() +# else /* C code */ +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK PyGILState_STATE _swig_thread_block = PyGILState_Ensure() +# define SWIG_PYTHON_THREAD_END_BLOCK PyGILState_Release(_swig_thread_block) +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW PyThreadState *_swig_thread_allow = PyEval_SaveThread() +# define SWIG_PYTHON_THREAD_END_ALLOW PyEval_RestoreThread(_swig_thread_allow) +# endif +# else /* Old thread way, not implemented, user must provide it */ +# if !defined(SWIG_PYTHON_INITIALIZE_THREADS) +# define SWIG_PYTHON_INITIALIZE_THREADS +# endif +# if !defined(SWIG_PYTHON_THREAD_BEGIN_BLOCK) +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK +# endif +# if !defined(SWIG_PYTHON_THREAD_END_BLOCK) +# define SWIG_PYTHON_THREAD_END_BLOCK +# endif +# if !defined(SWIG_PYTHON_THREAD_BEGIN_ALLOW) +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW +# endif +# if !defined(SWIG_PYTHON_THREAD_END_ALLOW) +# define SWIG_PYTHON_THREAD_END_ALLOW +# endif +# endif +#else /* No thread support */ +# define SWIG_PYTHON_INITIALIZE_THREADS +# define SWIG_PYTHON_THREAD_BEGIN_BLOCK +# define SWIG_PYTHON_THREAD_END_BLOCK +# define SWIG_PYTHON_THREAD_BEGIN_ALLOW +# define SWIG_PYTHON_THREAD_END_ALLOW +#endif + +/* ----------------------------------------------------------------------------- + * Python API portion that goes into the runtime + * ----------------------------------------------------------------------------- */ + +#ifdef __cplusplus +extern "C" { +#endif + +/* ----------------------------------------------------------------------------- + * Constant declarations + * ----------------------------------------------------------------------------- */ + +/* Constant Types */ +#define SWIG_PY_POINTER 4 +#define SWIG_PY_BINARY 5 + +/* Constant information structure */ +typedef struct swig_const_info { + int type; + const char *name; + long lvalue; + double dvalue; + void *pvalue; + swig_type_info **ptype; +} swig_const_info; + +#ifdef __cplusplus +} +#endif + + +/* ----------------------------------------------------------------------------- + * pyrun.swg + * + * This file contains the runtime support for Python modules + * and includes code for managing global variables and pointer + * type checking. + * + * ----------------------------------------------------------------------------- */ + +#if PY_VERSION_HEX < 0x02070000 /* 2.7.0 */ +# error "This version of SWIG only supports Python >= 2.7" +#endif + +#if PY_VERSION_HEX >= 0x03000000 && PY_VERSION_HEX < 0x03030000 +# error "This version of SWIG only supports Python 3 >= 3.3" +#endif + +/* Common SWIG API */ + +/* for raw pointers */ +#define SWIG_Python_ConvertPtr(obj, pptr, type, flags) SWIG_Python_ConvertPtrAndOwn(obj, pptr, type, flags, 0) +#define SWIG_ConvertPtr(obj, pptr, type, flags) SWIG_Python_ConvertPtr(obj, pptr, type, flags) +#define SWIG_ConvertPtrAndOwn(obj,pptr,type,flags,own) SWIG_Python_ConvertPtrAndOwn(obj, pptr, type, flags, own) + +#ifdef SWIGPYTHON_BUILTIN +#define SWIG_NewPointerObj(ptr, type, flags) SWIG_Python_NewPointerObj(self, ptr, type, flags) +#else +#define SWIG_NewPointerObj(ptr, type, flags) SWIG_Python_NewPointerObj(NULL, ptr, type, flags) +#endif + +#define SWIG_InternalNewPointerObj(ptr, type, flags) SWIG_Python_NewPointerObj(NULL, ptr, type, flags) + +#define SWIG_CheckImplicit(ty) SWIG_Python_CheckImplicit(ty) +#define SWIG_AcquirePtr(ptr, src) SWIG_Python_AcquirePtr(ptr, src) +#define swig_owntype int + +/* for raw packed data */ +#define SWIG_ConvertPacked(obj, ptr, sz, ty) SWIG_Python_ConvertPacked(obj, ptr, sz, ty) +#define SWIG_NewPackedObj(ptr, sz, type) SWIG_Python_NewPackedObj(ptr, sz, type) + +/* for class or struct pointers */ +#define SWIG_ConvertInstance(obj, pptr, type, flags) SWIG_ConvertPtr(obj, pptr, type, flags) +#define SWIG_NewInstanceObj(ptr, type, flags) SWIG_NewPointerObj(ptr, type, flags) + +/* for C or C++ function pointers */ +#define SWIG_ConvertFunctionPtr(obj, pptr, type) SWIG_Python_ConvertFunctionPtr(obj, pptr, type) +#define SWIG_NewFunctionPtrObj(ptr, type) SWIG_Python_NewPointerObj(NULL, ptr, type, 0) + +/* for C++ member pointers, ie, member methods */ +#define SWIG_ConvertMember(obj, ptr, sz, ty) SWIG_Python_ConvertPacked(obj, ptr, sz, ty) +#define SWIG_NewMemberObj(ptr, sz, type) SWIG_Python_NewPackedObj(ptr, sz, type) + + +/* Runtime API */ + +#define SWIG_GetModule(clientdata) SWIG_Python_GetModule(clientdata) +#define SWIG_SetModule(clientdata, pointer) SWIG_Python_SetModule(pointer) +#define SWIG_NewClientData(obj) SwigPyClientData_New(obj) + +#define SWIG_SetErrorObj SWIG_Python_SetErrorObj +#define SWIG_SetErrorMsg SWIG_Python_SetErrorMsg +#define SWIG_ErrorType(code) SWIG_Python_ErrorType(code) +#define SWIG_Error(code, msg) SWIG_Python_SetErrorMsg(SWIG_ErrorType(code), msg) +#define SWIG_fail goto fail + + +/* Runtime API implementation */ + +/* Error manipulation */ + +SWIGINTERN void +SWIG_Python_SetErrorObj(PyObject *errtype, PyObject *obj) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + PyErr_SetObject(errtype, obj); + SWIG_Py_DECREF(obj); + SWIG_PYTHON_THREAD_END_BLOCK; +} + +SWIGINTERN void +SWIG_Python_SetErrorMsg(PyObject *errtype, const char *msg) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + PyErr_SetString(errtype, msg); + SWIG_PYTHON_THREAD_END_BLOCK; +} + +#define SWIG_Python_Raise(obj, type, desc) SWIG_Python_SetErrorObj(SWIG_Python_ExceptionType(desc), obj) + +/* Set a constant value */ + +#if defined(SWIGPYTHON_BUILTIN) + +SWIGINTERN void +SwigPyBuiltin_AddPublicSymbol(PyObject *seq, const char *key) { + PyObject *s = PyString_InternFromString(key); + PyList_Append(seq, s); + SWIG_Py_DECREF(s); +} + +SWIGINTERN void +SWIG_Python_SetConstant(PyObject *d, PyObject *public_interface, const char *name, PyObject *obj) { + PyDict_SetItemString(d, name, obj); + SWIG_Py_DECREF(obj); + if (public_interface) + SwigPyBuiltin_AddPublicSymbol(public_interface, name); +} + +#else + +SWIGINTERN void +SWIG_Python_SetConstant(PyObject *d, const char *name, PyObject *obj) { + PyDict_SetItemString(d, name, obj); + SWIG_Py_DECREF(obj); +} + +#endif + +/* Append a value to the result obj */ + +SWIGINTERN PyObject* +SWIG_Python_AppendOutput(PyObject* result, PyObject* obj, int is_void) { + if (!result) { + result = obj; + } else if (result == Py_None && is_void) { + SWIG_Py_DECREF(result); + result = obj; + } else { + if (!PyList_Check(result)) { + PyObject *o2 = result; + result = PyList_New(1); + if (result) { + PyList_SET_ITEM(result, 0, o2); + } else { + SWIG_Py_DECREF(obj); + return o2; + } + } + PyList_Append(result,obj); + SWIG_Py_DECREF(obj); + } + return result; +} + +/* Unpack the argument tuple */ + +SWIGINTERN Py_ssize_t +SWIG_Python_UnpackTuple(PyObject *args, const char *name, Py_ssize_t min, Py_ssize_t max, PyObject **objs) +{ + if (!args) { + if (!min && !max) { + return 1; + } else { + PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got none", + name, (min == max ? "" : "at least "), (int)min); + return 0; + } + } + if (!PyTuple_Check(args)) { + if (min <= 1 && max >= 1) { + Py_ssize_t i; + objs[0] = args; + for (i = 1; i < max; ++i) { + objs[i] = 0; + } + return 2; + } + PyErr_SetString(PyExc_SystemError, "UnpackTuple() argument list is not a tuple"); + return 0; + } else { + Py_ssize_t l = PyTuple_GET_SIZE(args); + if (l < min) { + PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d", + name, (min == max ? "" : "at least "), (int)min, (int)l); + return 0; + } else if (l > max) { + PyErr_Format(PyExc_TypeError, "%s expected %s%d arguments, got %d", + name, (min == max ? "" : "at most "), (int)max, (int)l); + return 0; + } else { + Py_ssize_t i; + for (i = 0; i < l; ++i) { + objs[i] = PyTuple_GET_ITEM(args, i); + } + for (; l < max; ++l) { + objs[l] = 0; + } + return i + 1; + } + } +} + +SWIGINTERN int +SWIG_Python_CheckNoKeywords(PyObject *kwargs, const char *name) { + int no_kwargs = 1; + if (kwargs) { + assert(PyDict_Check(kwargs)); + if (PyDict_Size(kwargs) > 0) { + PyErr_Format(PyExc_TypeError, "%s() does not take keyword arguments", name); + no_kwargs = 0; + } + } + return no_kwargs; +} + +/* A functor is a function object with one single object argument */ +#define SWIG_Python_CallFunctor(functor, obj) PyObject_CallFunctionObjArgs(functor, obj, NULL); + +/* + Helper for static pointer initialization for both C and C++ code, for example + static PyObject *SWIG_STATIC_POINTER(MyVar) = NewSomething(...); +*/ +#ifdef __cplusplus +#define SWIG_STATIC_POINTER(var) var +#else +#define SWIG_STATIC_POINTER(var) var = 0; if (!var) var +#endif + +#ifdef __cplusplus +extern "C" { +#endif + +/* Python-specific SWIG API */ +#define SWIG_newvarlink() SWIG_Python_newvarlink() +#define SWIG_addvarlink(p, name, get_attr, set_attr) SWIG_Python_addvarlink(p, name, get_attr, set_attr) +#define SWIG_InstallConstants(d, constants) SWIG_Python_InstallConstants(d, constants) + +/* ----------------------------------------------------------------------------- + * global variable support code. + * ----------------------------------------------------------------------------- */ + +typedef struct swig_globalvar { + char *name; /* Name of global variable */ + PyObject *(*get_attr)(void); /* Return the current value */ + int (*set_attr)(PyObject *); /* Set the value */ + struct swig_globalvar *next; +} swig_globalvar; + +typedef struct swig_varlinkobject { + PyObject_HEAD + swig_globalvar *vars; +} swig_varlinkobject; + +SWIGINTERN PyObject * +swig_varlink_repr(PyObject *SWIGUNUSEDPARM(v)) { +#if PY_VERSION_HEX >= 0x03000000 + return PyUnicode_InternFromString(""); +#else + return PyString_FromString(""); +#endif +} + +SWIGINTERN PyObject * +swig_varlink_str(PyObject *o) { + swig_varlinkobject *v = (swig_varlinkobject *) o; +#if PY_VERSION_HEX >= 0x03000000 + PyObject *str = PyUnicode_InternFromString("("); + PyObject *tail; + PyObject *joined; + swig_globalvar *var; + for (var = v->vars; var; var=var->next) { + tail = PyUnicode_FromString(var->name); + joined = PyUnicode_Concat(str, tail); + SWIG_Py_DECREF(str); + SWIG_Py_DECREF(tail); + str = joined; + if (var->next) { + tail = PyUnicode_InternFromString(", "); + joined = PyUnicode_Concat(str, tail); + SWIG_Py_DECREF(str); + SWIG_Py_DECREF(tail); + str = joined; + } + } + tail = PyUnicode_InternFromString(")"); + joined = PyUnicode_Concat(str, tail); + SWIG_Py_DECREF(str); + SWIG_Py_DECREF(tail); + str = joined; +#else + PyObject *str = PyString_FromString("("); + swig_globalvar *var; + for (var = v->vars; var; var=var->next) { + PyString_ConcatAndDel(&str,PyString_FromString(var->name)); + if (var->next) PyString_ConcatAndDel(&str,PyString_FromString(", ")); + } + PyString_ConcatAndDel(&str,PyString_FromString(")")); +#endif + return str; +} + +SWIGINTERN void +swig_varlink_dealloc(PyObject *o) { + swig_varlinkobject *v = (swig_varlinkobject *) o; + swig_globalvar *var = v->vars; + while (var) { + swig_globalvar *n = var->next; + free(var->name); + free(var); + var = n; + } +} + +SWIGINTERN PyObject * +swig_varlink_getattr(PyObject *o, char *n) { + swig_varlinkobject *v = (swig_varlinkobject *) o; + PyObject *res = NULL; + swig_globalvar *var = v->vars; + while (var) { + if (strcmp(var->name,n) == 0) { + res = (*var->get_attr)(); + break; + } + var = var->next; + } + if (res == NULL && !PyErr_Occurred()) { + PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n); + } + return res; +} + +SWIGINTERN int +swig_varlink_setattr(PyObject *o, char *n, PyObject *p) { + swig_varlinkobject *v = (swig_varlinkobject *) o; + int res = 1; + swig_globalvar *var = v->vars; + while (var) { + if (strcmp(var->name,n) == 0) { + res = (*var->set_attr)(p); + break; + } + var = var->next; + } + if (res == 1 && !PyErr_Occurred()) { + PyErr_Format(PyExc_AttributeError, "Unknown C global variable '%s'", n); + } + return res; +} + +SWIGINTERN PyTypeObject* +swig_varlink_type(void) { + static char varlink__doc__[] = "Swig var link object"; +#ifndef SWIG_HEAPTYPES + static PyTypeObject varlink_type; + static int type_init = 0; + if (!type_init) { + const PyTypeObject tmp = { +#if PY_VERSION_HEX >= 0x03000000 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "swigvarlink", /* tp_name */ + sizeof(swig_varlinkobject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor) swig_varlink_dealloc, /* tp_dealloc */ +#if PY_VERSION_HEX < 0x030800b4 + (printfunc)0, /* tp_print */ +#else + (Py_ssize_t)0, /* tp_vectorcall_offset */ +#endif + (getattrfunc) swig_varlink_getattr, /* tp_getattr */ + (setattrfunc) swig_varlink_setattr, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc) swig_varlink_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + (reprfunc) swig_varlink_str, /* tp_str */ + 0, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + 0, /* tp_flags */ + varlink__doc__, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, /* tp_iter -> tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x03040000 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#if (PY_VERSION_HEX >= 0x03080000) && (PY_VERSION_HEX < 0x03090000) + 0, /* tp_print */ +#endif +#if PY_VERSION_HEX >= 0x030c0000 + 0, /* tp_watched */ +#endif +#if PY_VERSION_HEX >= 0x030d00a4 + 0, /* tp_versions_used */ +#endif +#ifdef COUNT_ALLOCS + 0, /* tp_allocs */ + 0, /* tp_frees */ + 0, /* tp_maxalloc */ + 0, /* tp_prev */ + 0 /* tp_next */ +#endif + }; + varlink_type = tmp; + type_init = 1; + if (PyType_Ready(&varlink_type) < 0) + return NULL; + } + return &varlink_type; +#else + PyType_Slot slots[] = { + { Py_tp_dealloc, (void *)swig_varlink_dealloc }, + { Py_tp_repr, (void *)swig_varlink_repr }, + { Py_tp_getattr, (void *)swig_varlink_getattr }, + { Py_tp_setattr, (void *)swig_varlink_setattr }, + { Py_tp_str, (void *)swig_varlink_str }, + { Py_tp_doc, (void *)varlink__doc__ }, + { 0, NULL } + }; + PyType_Spec spec = { + "swigvarlink", + sizeof(swig_varlinkobject), + 0, + Py_TPFLAGS_DEFAULT, + slots + }; + return (PyTypeObject *)PyType_FromSpec(&spec); +#endif +} + +/* Create a variable linking object for use later */ +SWIGINTERN PyObject * +SWIG_Python_newvarlink(void) { + swig_varlinkobject *result = PyObject_New(swig_varlinkobject, swig_varlink_type()); + if (result) { + result->vars = 0; + } + return ((PyObject*) result); +} + +SWIGINTERN void +SWIG_Python_addvarlink(PyObject *p, const char *name, PyObject *(*get_attr)(void), int (*set_attr)(PyObject *p)) { + swig_varlinkobject *v = (swig_varlinkobject *) p; + swig_globalvar *gv = (swig_globalvar *) malloc(sizeof(swig_globalvar)); + if (gv) { + size_t size = strlen(name)+1; + gv->name = (char *)malloc(size); + if (gv->name) { + memcpy(gv->name, name, size); + gv->get_attr = get_attr; + gv->set_attr = set_attr; + gv->next = v->vars; + } + } + v->vars = gv; +} + + +static PyObject *Swig_Globals_global = NULL; + +SWIGINTERN PyObject * +SWIG_globals(void) { + if (Swig_Globals_global == NULL) { + Swig_Globals_global = SWIG_newvarlink(); + } + return Swig_Globals_global; +} + +#ifdef __cplusplus +} +#endif + +/* ----------------------------------------------------------------------------- + * Pointer declarations + * ----------------------------------------------------------------------------- */ + +/* Flags for new pointer objects */ +#define SWIG_POINTER_NOSHADOW (SWIG_POINTER_OWN << 1) +#define SWIG_POINTER_NEW (SWIG_POINTER_NOSHADOW | SWIG_POINTER_OWN) + +#define SWIG_POINTER_IMPLICIT_CONV (SWIG_POINTER_DISOWN << 1) + +#define SWIG_BUILTIN_TP_INIT (SWIG_POINTER_OWN << 2) +#define SWIG_BUILTIN_INIT (SWIG_BUILTIN_TP_INIT | SWIG_POINTER_OWN) + +#ifdef __cplusplus +extern "C" { +#endif + +/* The python void return value */ + +SWIGRUNTIMEINLINE PyObject * +SWIG_Py_Void(void) +{ + PyObject *none = Py_None; + SWIG_Py_INCREF(none); + return none; +} + +/* SwigPyClientData */ + +typedef struct { + PyObject *klass; + PyObject *newraw; + PyObject *newargs; + PyObject *destroy; + int delargs; + int implicitconv; + PyTypeObject *pytype; +} SwigPyClientData; + +SWIGRUNTIMEINLINE int +SWIG_Python_CheckImplicit(swig_type_info *ty) +{ + SwigPyClientData *data = (SwigPyClientData *)ty->clientdata; + int fail = data ? data->implicitconv : 0; + if (fail) + PyErr_SetString(PyExc_TypeError, "Implicit conversion is prohibited for explicit constructors."); + return fail; +} + +SWIGRUNTIMEINLINE PyObject * +SWIG_Python_ExceptionType(swig_type_info *desc) { + SwigPyClientData *data = desc ? (SwigPyClientData *) desc->clientdata : 0; + PyObject *klass = data ? data->klass : 0; + return (klass ? klass : PyExc_RuntimeError); +} + + +SWIGRUNTIME SwigPyClientData * +SwigPyClientData_New(PyObject* obj) +{ + if (!obj) { + return 0; + } else { + SwigPyClientData *data = (SwigPyClientData *)malloc(sizeof(SwigPyClientData)); + /* the klass element */ + data->klass = obj; + SWIG_Py_INCREF(data->klass); + /* the newraw method and newargs arguments used to create a new raw instance */ + if (PyClass_Check(obj)) { + data->newraw = 0; + SWIG_Py_INCREF(obj); + data->newargs = obj; + } else { + data->newraw = PyObject_GetAttrString(data->klass, "__new__"); + if (data->newraw) { + data->newargs = PyTuple_New(1); + if (data->newargs) { + SWIG_Py_INCREF(obj); + PyTuple_SET_ITEM(data->newargs, 0, obj); + } else { + SWIG_Py_DECREF(data->newraw); + SWIG_Py_DECREF(data->klass); + free(data); + return 0; + } + } else { + SWIG_Py_INCREF(obj); + data->newargs = obj; + } + } + /* the destroy method, aka as the C++ delete method */ + data->destroy = PyObject_GetAttrString(data->klass, "__swig_destroy__"); + if (PyErr_Occurred()) { + PyErr_Clear(); + data->destroy = 0; + } + if (data->destroy) { + data->delargs = !(PyCFunction_GET_FLAGS(data->destroy) & METH_O); + } else { + data->delargs = 0; + } + data->implicitconv = 0; + data->pytype = 0; + return data; + } +} + +SWIGRUNTIME void +SwigPyClientData_Del(SwigPyClientData *data) +{ + SWIG_Py_XDECREF(data->klass); + SWIG_Py_XDECREF(data->newraw); + SWIG_Py_XDECREF(data->newargs); + SWIG_Py_XDECREF(data->destroy); + free(data); +} + +/* =============== SwigPyObject =====================*/ + +typedef struct { + PyObject_HEAD + void *ptr; + swig_type_info *ty; + int own; + PyObject *next; +#ifdef SWIGPYTHON_BUILTIN + PyObject *dict; +#endif +} SwigPyObject; + + +#ifdef SWIGPYTHON_BUILTIN + +SWIGRUNTIME PyObject * +SwigPyObject_get___dict__(PyObject *v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *)v; + + if (!sobj->dict) + sobj->dict = PyDict_New(); + + SWIG_Py_XINCREF(sobj->dict); + return sobj->dict; +} + +#endif + +SWIGRUNTIME PyObject * +SwigPyObject_long(SwigPyObject *v) +{ + return PyLong_FromVoidPtr(v->ptr); +} + +SWIGRUNTIME PyObject * +SwigPyObject_format(const char* fmt, SwigPyObject *v) +{ + PyObject *res = NULL; + PyObject *args = PyTuple_New(1); + if (args) { + PyObject *val = SwigPyObject_long(v); + if (val) { + PyObject *ofmt; + PyTuple_SET_ITEM(args, 0, val); + ofmt = SWIG_Python_str_FromChar(fmt); + if (ofmt) { +#if PY_VERSION_HEX >= 0x03000000 + res = PyUnicode_Format(ofmt,args); +#else + res = PyString_Format(ofmt,args); +#endif + SWIG_Py_DECREF(ofmt); + } + } + SWIG_Py_DECREF(args); + } + return res; +} + +SWIGRUNTIME PyObject * +SwigPyObject_oct(SwigPyObject *v) +{ + return SwigPyObject_format("%o",v); +} + +SWIGRUNTIME PyObject * +SwigPyObject_hex(SwigPyObject *v) +{ + return SwigPyObject_format("%x",v); +} + +SWIGRUNTIME PyObject * +SwigPyObject_repr(SwigPyObject *v) +{ + const char *name = SWIG_TypePrettyName(v->ty); + PyObject *repr = SWIG_Python_str_FromFormat("", (name ? name : "unknown"), (void *)v); + if (repr && v->next) { + PyObject *nrep = SwigPyObject_repr((SwigPyObject *)v->next); + if (nrep) { +# if PY_VERSION_HEX >= 0x03000000 + PyObject *joined = PyUnicode_Concat(repr, nrep); + SWIG_Py_DECREF(repr); + SWIG_Py_DECREF(nrep); + repr = joined; +# else + PyString_ConcatAndDel(&repr,nrep); +# endif + } else { + SWIG_Py_DECREF(repr); + repr = NULL; + } + } + return repr; +} + +/* We need a version taking two PyObject* parameters so it's a valid + * PyCFunction to use in swigobject_methods[]. */ +SWIGRUNTIME PyObject * +SwigPyObject_repr2(PyObject *v, PyObject *SWIGUNUSEDPARM(args)) +{ + return SwigPyObject_repr((SwigPyObject*)v); +} + +SWIGRUNTIME int +SwigPyObject_compare(SwigPyObject *v, SwigPyObject *w) +{ + void *i = v->ptr; + void *j = w->ptr; + return (i < j) ? -1 : ((i > j) ? 1 : 0); +} + +/* Added for Python 3.x, would it also be useful for Python 2.x? */ +SWIGRUNTIME PyObject* +SwigPyObject_richcompare(SwigPyObject *v, SwigPyObject *w, int op) +{ + PyObject* res = NULL; + if (!PyErr_Occurred()) { + if (op != Py_EQ && op != Py_NE) { + SWIG_Py_INCREF(Py_NotImplemented); + return Py_NotImplemented; + } + res = PyBool_FromLong( (SwigPyObject_compare(v, w)==0) == (op == Py_EQ) ? 1 : 0); + } + return res; +} + + +SWIGRUNTIME PyTypeObject* SwigPyObject_TypeOnce(void); + +#ifdef SWIGPYTHON_BUILTIN +static swig_type_info *SwigPyObject_stype = 0; +SWIGRUNTIME PyTypeObject* +SwigPyObject_type(void) { + SwigPyClientData *cd; + assert(SwigPyObject_stype); + cd = (SwigPyClientData*) SwigPyObject_stype->clientdata; + assert(cd); + assert(cd->pytype); + return cd->pytype; +} +#else +SWIGRUNTIME PyTypeObject* +SwigPyObject_type(void) { + static PyTypeObject *SWIG_STATIC_POINTER(type) = SwigPyObject_TypeOnce(); + return type; +} +#endif + +SWIGRUNTIMEINLINE int +SwigPyObject_Check(PyObject *op) { + PyTypeObject *target_tp = SwigPyObject_type(); + PyTypeObject *op_type = Py_TYPE(op); +#ifdef SWIGPYTHON_BUILTIN + if (PyType_IsSubtype(op_type, target_tp)) + return 1; + return (strcmp(op_type->tp_name, "SwigPyObject") == 0); +#else +# ifdef Py_LIMITED_API + int cmp; + PyObject *tp_name; +#endif + if (op_type == target_tp) + return 1; +# ifdef Py_LIMITED_API + tp_name = PyObject_GetAttrString((PyObject *)op_type, "__name__"); + if (!tp_name) + return 0; + cmp = PyUnicode_CompareWithASCIIString(tp_name, "SwigPyObject"); + SWIG_Py_DECREF(tp_name); + return cmp == 0; +# else + return (strcmp(op_type->tp_name, "SwigPyObject") == 0); +# endif +#endif +} + +SWIGRUNTIME PyObject * +SwigPyObject_New(void *ptr, swig_type_info *ty, int own); + +static PyObject* Swig_Capsule_global = NULL; + +SWIGRUNTIME void +SwigPyObject_dealloc(PyObject *v) +{ + SwigPyObject *sobj = (SwigPyObject *) v; + PyObject *next = sobj->next; + if (sobj->own == SWIG_POINTER_OWN) { + swig_type_info *ty = sobj->ty; + SwigPyClientData *data = ty ? (SwigPyClientData *) ty->clientdata : 0; + PyObject *destroy = data ? data->destroy : 0; + if (destroy) { + /* destroy is always a VARARGS method */ + PyObject *res; + + /* PyObject_CallFunction() has the potential to silently drop + the active exception. In cases of unnamed temporary + variable or where we just finished iterating over a generator + StopIteration will be active right now, and this needs to + remain true upon return from SwigPyObject_dealloc. So save + and restore. */ + + PyObject *type = NULL, *value = NULL, *traceback = NULL; + PyErr_Fetch(&type, &value, &traceback); + + if (data->delargs) { + /* we need to create a temporary object to carry the destroy operation */ + PyObject *tmp = SwigPyObject_New(sobj->ptr, ty, 0); + if (tmp) { + res = SWIG_Python_CallFunctor(destroy, tmp); + } else { + res = 0; + } + SWIG_Py_XDECREF(tmp); + } else { + PyCFunction meth = PyCFunction_GET_FUNCTION(destroy); + PyObject *mself = PyCFunction_GET_SELF(destroy); + res = ((*meth)(mself, v)); + } + if (!res) + PyErr_WriteUnraisable(destroy); + + PyErr_Restore(type, value, traceback); + + SWIG_Py_XDECREF(res); + } +#if !defined(SWIG_PYTHON_SILENT_MEMLEAK) + else { + const char *name = SWIG_TypePrettyName(ty); + printf("swig/python detected a memory leak of type '%s', no destructor found.\n", (name ? name : "unknown")); + } +#endif + SWIG_Py_XDECREF(Swig_Capsule_global); + } + SWIG_Py_XDECREF(next); +#ifdef SWIGPYTHON_BUILTIN + SWIG_Py_XDECREF(sobj->dict); +#endif + PyObject_Free(v); +} + +SWIGRUNTIME PyObject* +SwigPyObject_append(PyObject* v, PyObject* next) +{ + SwigPyObject *sobj = (SwigPyObject *) v; + if (!SwigPyObject_Check(next)) { + PyErr_SetString(PyExc_TypeError, "Attempt to append a non SwigPyObject"); + return NULL; + } + ((SwigPyObject *)next)->next = sobj->next; + sobj->next = next; + SWIG_Py_INCREF(next); + return SWIG_Py_Void(); +} + +SWIGRUNTIME PyObject* +SwigPyObject_next(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *) v; + if (sobj->next) { + SWIG_Py_INCREF(sobj->next); + return sobj->next; + } else { + return SWIG_Py_Void(); + } +} + +SWIGINTERN PyObject* +SwigPyObject_disown(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *)v; + sobj->own = 0; + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject* +SwigPyObject_acquire(PyObject* v, PyObject *SWIGUNUSEDPARM(args)) +{ + SwigPyObject *sobj = (SwigPyObject *)v; + sobj->own = SWIG_POINTER_OWN; + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject* +SwigPyObject_own(PyObject *v, PyObject *args) +{ + PyObject *val = 0; + if (!PyArg_UnpackTuple(args, "own", 0, 1, &val)) { + return NULL; + } else { + SwigPyObject *sobj = (SwigPyObject *)v; + PyObject *obj = PyBool_FromLong(sobj->own); + if (val) { + if (PyObject_IsTrue(val)) { + SWIG_Py_DECREF(SwigPyObject_acquire(v,args)); + } else { + SWIG_Py_DECREF(SwigPyObject_disown(v,args)); + } + } + return obj; + } +} + +static PyMethodDef +swigobject_methods[] = { + {"disown", SwigPyObject_disown, METH_NOARGS, "releases ownership of the pointer"}, + {"acquire", SwigPyObject_acquire, METH_NOARGS, "acquires ownership of the pointer"}, + {"own", SwigPyObject_own, METH_VARARGS, "returns/sets ownership of the pointer"}, + {"append", SwigPyObject_append, METH_O, "appends another 'this' object"}, + {"next", SwigPyObject_next, METH_NOARGS, "returns the next 'this' object"}, + {"__repr__",SwigPyObject_repr2, METH_NOARGS, "returns object representation"}, + {0, 0, 0, 0} +}; + +SWIGRUNTIME PyTypeObject* +SwigPyObject_TypeOnce(void) { + static char swigobject_doc[] = "Swig object carries a C/C++ instance pointer"; +#ifndef SWIG_HEAPTYPES + static PyNumberMethods SwigPyObject_as_number = { + (binaryfunc)0, /*nb_add*/ + (binaryfunc)0, /*nb_subtract*/ + (binaryfunc)0, /*nb_multiply*/ + /* nb_divide removed in Python 3 */ +#if PY_VERSION_HEX < 0x03000000 + (binaryfunc)0, /*nb_divide*/ +#endif + (binaryfunc)0, /*nb_remainder*/ + (binaryfunc)0, /*nb_divmod*/ + (ternaryfunc)0,/*nb_power*/ + (unaryfunc)0, /*nb_negative*/ + (unaryfunc)0, /*nb_positive*/ + (unaryfunc)0, /*nb_absolute*/ + (inquiry)0, /*nb_nonzero*/ + 0, /*nb_invert*/ + 0, /*nb_lshift*/ + 0, /*nb_rshift*/ + 0, /*nb_and*/ + 0, /*nb_xor*/ + 0, /*nb_or*/ +#if PY_VERSION_HEX < 0x03000000 + 0, /*nb_coerce*/ +#endif + (unaryfunc)SwigPyObject_long, /*nb_int*/ +#if PY_VERSION_HEX < 0x03000000 + (unaryfunc)SwigPyObject_long, /*nb_long*/ +#else + 0, /*nb_reserved*/ +#endif + (unaryfunc)0, /*nb_float*/ +#if PY_VERSION_HEX < 0x03000000 + (unaryfunc)SwigPyObject_oct, /*nb_oct*/ + (unaryfunc)SwigPyObject_hex, /*nb_hex*/ +#endif +#if PY_VERSION_HEX >= 0x03050000 /* 3.5 */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_inplace_matrix_multiply */ +#elif PY_VERSION_HEX >= 0x03000000 /* 3.0 */ + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_index, nb_inplace_divide removed */ +#else + 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 /* nb_inplace_add -> nb_index */ +#endif + }; + + static PyTypeObject swigpyobject_type; + static int type_init = 0; + if (!type_init) { + const PyTypeObject tmp = { +#if PY_VERSION_HEX >= 0x03000000 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "SwigPyObject", /* tp_name */ + sizeof(SwigPyObject), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)SwigPyObject_dealloc, /* tp_dealloc */ +#if PY_VERSION_HEX < 0x030800b4 + (printfunc)0, /* tp_print */ +#else + (Py_ssize_t)0, /* tp_vectorcall_offset */ +#endif + (getattrfunc)0, /* tp_getattr */ + (setattrfunc)0, /* tp_setattr */ +#if PY_VERSION_HEX >= 0x03000000 + 0, /* tp_reserved in 3.0.1, tp_compare in 3.0.0 but not used */ +#else + (cmpfunc)SwigPyObject_compare, /* tp_compare */ +#endif + (reprfunc)SwigPyObject_repr, /* tp_repr */ + &SwigPyObject_as_number, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + (hashfunc)0, /* tp_hash */ + (ternaryfunc)0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + swigobject_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + (richcmpfunc)SwigPyObject_richcompare,/* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + swigobject_methods, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x03040000 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#if (PY_VERSION_HEX >= 0x03080000) && (PY_VERSION_HEX < 0x03090000) + 0, /* tp_print */ +#endif +#if PY_VERSION_HEX >= 0x030c0000 + 0, /* tp_watched */ +#endif +#if PY_VERSION_HEX >= 0x030d00a4 + 0, /* tp_versions_used */ +#endif +#ifdef COUNT_ALLOCS + 0, /* tp_allocs */ + 0, /* tp_frees */ + 0, /* tp_maxalloc */ + 0, /* tp_prev */ + 0 /* tp_next */ +#endif + }; + swigpyobject_type = tmp; + type_init = 1; + if (PyType_Ready(&swigpyobject_type) != 0) + return NULL; + } + return &swigpyobject_type; +#else + PyType_Slot slots[] = { + { Py_tp_dealloc, (void *)SwigPyObject_dealloc }, + { Py_tp_repr, (void *)SwigPyObject_repr }, + { Py_tp_getattro, (void *)PyObject_GenericGetAttr }, + { Py_tp_doc, (void *)swigobject_doc }, + { Py_tp_richcompare, (void *)SwigPyObject_richcompare }, + { Py_tp_methods, (void *)swigobject_methods }, + { Py_nb_int, (void *)SwigPyObject_long }, + { 0, NULL } + }; + PyType_Spec spec = { + "SwigPyObject", + sizeof(SwigPyObject), + 0, + Py_TPFLAGS_DEFAULT|Py_TPFLAGS_BASETYPE, + slots + }; + return (PyTypeObject *)PyType_FromSpec(&spec); +#endif +} + +SWIGRUNTIME PyObject * +SwigPyObject_New(void *ptr, swig_type_info *ty, int own) +{ + SwigPyObject *sobj = PyObject_New(SwigPyObject, SwigPyObject_type()); + if (sobj) { + sobj->ptr = ptr; + sobj->ty = ty; + sobj->own = own; + sobj->next = 0; +#ifdef SWIGPYTHON_BUILTIN + sobj->dict = 0; +#endif + if (own == SWIG_POINTER_OWN) { + /* Obtain a reference to the Python capsule wrapping the module information, so that the + * module information is correctly destroyed after all SWIG python objects have been freed + * by the GC (and corresponding destructors invoked) */ + SWIG_Py_XINCREF(Swig_Capsule_global); + } + } + return (PyObject *)sobj; +} + +/* ----------------------------------------------------------------------------- + * Implements a simple Swig Packed type, and use it instead of string + * ----------------------------------------------------------------------------- */ + +typedef struct { + PyObject_HEAD + void *pack; + swig_type_info *ty; + size_t size; +} SwigPyPacked; + +SWIGRUNTIME PyObject * +SwigPyPacked_repr(SwigPyPacked *v) +{ + char result[SWIG_BUFFER_SIZE]; + if (SWIG_PackDataName(result, v->pack, v->size, 0, sizeof(result))) { + return SWIG_Python_str_FromFormat("", result, v->ty->name); + } else { + return SWIG_Python_str_FromFormat("", v->ty->name); + } +} + +SWIGRUNTIME PyObject * +SwigPyPacked_str(SwigPyPacked *v) +{ + char result[SWIG_BUFFER_SIZE]; + if (SWIG_PackDataName(result, v->pack, v->size, 0, sizeof(result))){ + return SWIG_Python_str_FromFormat("%s%s", result, v->ty->name); + } else { + return SWIG_Python_str_FromChar(v->ty->name); + } +} + +SWIGRUNTIME int +SwigPyPacked_compare(SwigPyPacked *v, SwigPyPacked *w) +{ + size_t i = v->size; + size_t j = w->size; + int s = (i < j) ? -1 : ((i > j) ? 1 : 0); + return s ? s : strncmp((const char *)v->pack, (const char *)w->pack, 2*v->size); +} + +SWIGRUNTIME PyTypeObject* SwigPyPacked_TypeOnce(void); + +SWIGRUNTIME PyTypeObject* +SwigPyPacked_type(void) { + static PyTypeObject *SWIG_STATIC_POINTER(type) = SwigPyPacked_TypeOnce(); + return type; +} + +SWIGRUNTIMEINLINE int +SwigPyPacked_Check(PyObject *op) { +#ifdef Py_LIMITED_API + int cmp; + PyObject *tp_name; +#endif + PyTypeObject* op_type = Py_TYPE(op); + if (op_type == SwigPyPacked_TypeOnce()) + return 1; +#ifdef Py_LIMITED_API + tp_name = PyObject_GetAttrString((PyObject *)op_type, "__name__"); + if (!tp_name) + return 0; + cmp = PyUnicode_CompareWithASCIIString(tp_name, "SwigPyPacked"); + SWIG_Py_DECREF(tp_name); + return cmp == 0; +#else + return (strcmp(op_type->tp_name, "SwigPyPacked") == 0); +#endif +} + +SWIGRUNTIME void +SwigPyPacked_dealloc(PyObject *v) +{ + if (SwigPyPacked_Check(v)) { + SwigPyPacked *sobj = (SwigPyPacked *) v; + free(sobj->pack); + } + PyObject_Free(v); +} + +SWIGRUNTIME PyTypeObject* +SwigPyPacked_TypeOnce(void) { + static char swigpacked_doc[] = "Swig object carries a C/C++ instance pointer"; +#ifndef SWIG_HEAPTYPES + static PyTypeObject swigpypacked_type; + static int type_init = 0; + if (!type_init) { + const PyTypeObject tmp = { +#if PY_VERSION_HEX>=0x03000000 + PyVarObject_HEAD_INIT(NULL, 0) +#else + PyObject_HEAD_INIT(NULL) + 0, /* ob_size */ +#endif + "SwigPyPacked", /* tp_name */ + sizeof(SwigPyPacked), /* tp_basicsize */ + 0, /* tp_itemsize */ + (destructor)SwigPyPacked_dealloc, /* tp_dealloc */ +#if PY_VERSION_HEX < 0x030800b4 + (printfunc)0, /* tp_print */ +#else + (Py_ssize_t)0, /* tp_vectorcall_offset */ +#endif + (getattrfunc)0, /* tp_getattr */ + (setattrfunc)0, /* tp_setattr */ +#if PY_VERSION_HEX>=0x03000000 + 0, /* tp_reserved in 3.0.1 */ +#else + (cmpfunc)SwigPyPacked_compare, /* tp_compare */ +#endif + (reprfunc)SwigPyPacked_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + (hashfunc)0, /* tp_hash */ + (ternaryfunc)0, /* tp_call */ + (reprfunc)SwigPyPacked_str, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT, /* tp_flags */ + swigpacked_doc, /* tp_doc */ + 0, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + 0, /* tp_methods */ + 0, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + 0, /* tp_init */ + 0, /* tp_alloc */ + 0, /* tp_new */ + 0, /* tp_free */ + 0, /* tp_is_gc */ + 0, /* tp_bases */ + 0, /* tp_mro */ + 0, /* tp_cache */ + 0, /* tp_subclasses */ + 0, /* tp_weaklist */ + 0, /* tp_del */ + 0, /* tp_version_tag */ +#if PY_VERSION_HEX >= 0x03040000 + 0, /* tp_finalize */ +#endif +#if PY_VERSION_HEX >= 0x03080000 + 0, /* tp_vectorcall */ +#endif +#if (PY_VERSION_HEX >= 0x03080000) && (PY_VERSION_HEX < 0x03090000) + 0, /* tp_print */ +#endif +#if PY_VERSION_HEX >= 0x030c0000 + 0, /* tp_watched */ +#endif +#if PY_VERSION_HEX >= 0x030d00a4 + 0, /* tp_versions_used */ +#endif +#ifdef COUNT_ALLOCS + 0, /* tp_allocs */ + 0, /* tp_frees */ + 0, /* tp_maxalloc */ + 0, /* tp_prev */ + 0 /* tp_next */ +#endif + }; + swigpypacked_type = tmp; + type_init = 1; + if (PyType_Ready(&swigpypacked_type) != 0) + return NULL; + } + return &swigpypacked_type; +#else + PyType_Slot slots[] = { + { Py_tp_dealloc, (void *)SwigPyPacked_dealloc }, + { Py_tp_repr, (void *)SwigPyPacked_repr }, + { Py_tp_str, (void *)SwigPyPacked_str }, + { Py_tp_getattro, (void *)PyObject_GenericGetAttr }, + { Py_tp_doc, (void *)swigpacked_doc }, + { 0, NULL } + }; + PyType_Spec spec = { + "SwigPyPacked", + sizeof(SwigPyPacked), + 0, + Py_TPFLAGS_DEFAULT, + slots + }; + return (PyTypeObject *)PyType_FromSpec(&spec); +#endif +} + +SWIGRUNTIME PyObject * +SwigPyPacked_New(void *ptr, size_t size, swig_type_info *ty) +{ + SwigPyPacked *sobj = PyObject_New(SwigPyPacked, SwigPyPacked_type()); + if (sobj) { + void *pack = malloc(size); + if (pack) { + memcpy(pack, ptr, size); + sobj->pack = pack; + sobj->ty = ty; + sobj->size = size; + } else { + PyObject_Free((PyObject *)sobj); + sobj = 0; + } + } + return (PyObject *) sobj; +} + +SWIGRUNTIME swig_type_info * +SwigPyPacked_UnpackData(PyObject *obj, void *ptr, size_t size) +{ + if (SwigPyPacked_Check(obj)) { + SwigPyPacked *sobj = (SwigPyPacked *)obj; + if (sobj->size != size) return 0; + memcpy(ptr, sobj->pack, size); + return sobj->ty; + } else { + return 0; + } +} + +/* ----------------------------------------------------------------------------- + * pointers/data manipulation + * ----------------------------------------------------------------------------- */ + +static PyObject *Swig_This_global = NULL; + +SWIGRUNTIME PyObject * +SWIG_This(void) +{ + if (Swig_This_global == NULL) + Swig_This_global = SWIG_Python_str_FromChar("this"); + return Swig_This_global; +} + +/* #define SWIG_PYTHON_SLOW_GETSET_THIS */ + +/* TODO: I don't know how to implement the fast getset in Python 3 right now */ +#if PY_VERSION_HEX>=0x03000000 +#define SWIG_PYTHON_SLOW_GETSET_THIS +#endif + +SWIGRUNTIME SwigPyObject * +SWIG_Python_GetSwigThis(PyObject *pyobj) +{ + PyObject *obj; + + if (SwigPyObject_Check(pyobj)) + return (SwigPyObject *) pyobj; + +#ifdef SWIGPYTHON_BUILTIN + (void)obj; +# ifdef PyWeakref_CheckProxy + if (PyWeakref_CheckProxy(pyobj)) { +#if PY_VERSION_HEX >= 0x030d0000 + PyWeakref_GetRef(pyobj, &pyobj); + Py_DECREF(pyobj); +#else + pyobj = PyWeakref_GET_OBJECT(pyobj); +#endif + if (pyobj && SwigPyObject_Check(pyobj)) + return (SwigPyObject*) pyobj; + } +# endif + return NULL; +#else + + obj = 0; + +#if !defined(SWIG_PYTHON_SLOW_GETSET_THIS) + if (PyInstance_Check(pyobj)) { + obj = _PyInstance_Lookup(pyobj, SWIG_This()); + } else { + PyObject **dictptr = _PyObject_GetDictPtr(pyobj); + if (dictptr != NULL) { + PyObject *dict = *dictptr; + obj = dict ? PyDict_GetItem(dict, SWIG_This()) : 0; + } else { +#ifdef PyWeakref_CheckProxy + if (PyWeakref_CheckProxy(pyobj)) { + PyObject *wobj = PyWeakref_GET_OBJECT(pyobj); + return wobj ? SWIG_Python_GetSwigThis(wobj) : 0; + } +#endif + obj = PyObject_GetAttr(pyobj,SWIG_This()); + if (obj) { + SWIG_Py_DECREF(obj); + } else { + if (PyErr_Occurred()) PyErr_Clear(); + return 0; + } + } + } +#else + obj = PyObject_GetAttr(pyobj,SWIG_This()); + if (obj) { + SWIG_Py_DECREF(obj); + } else { + if (PyErr_Occurred()) PyErr_Clear(); + return 0; + } +#endif + if (obj && !SwigPyObject_Check(obj)) { + /* a PyObject is called 'this', try to get the 'real this' + SwigPyObject from it */ + return SWIG_Python_GetSwigThis(obj); + } + return (SwigPyObject *)obj; +#endif +} + +/* Acquire a pointer value */ + +SWIGRUNTIME int +SWIG_Python_AcquirePtr(PyObject *obj, int own) { + if (own == SWIG_POINTER_OWN) { + SwigPyObject *sobj = SWIG_Python_GetSwigThis(obj); + if (sobj) { + int oldown = sobj->own; + sobj->own = own; + return oldown; + } + } + return 0; +} + +/* Convert a pointer value */ + +SWIGRUNTIME int +SWIG_Python_ConvertPtrAndOwn(PyObject *obj, void **ptr, swig_type_info *ty, int flags, int *own) { + int res; + SwigPyObject *sobj; + int implicit_conv = (flags & SWIG_POINTER_IMPLICIT_CONV) != 0; + + if (!obj) + return SWIG_ERROR; + if (obj == Py_None && !implicit_conv) { + if (ptr) + *ptr = 0; + return (flags & SWIG_POINTER_NO_NULL) ? SWIG_NullReferenceError : SWIG_OK; + } + + res = SWIG_ERROR; + + sobj = SWIG_Python_GetSwigThis(obj); + if (own) + *own = 0; + while (sobj) { + void *vptr = sobj->ptr; + if (ty) { + swig_type_info *to = sobj->ty; + if (to == ty) { + /* no type cast needed */ + if (ptr) *ptr = vptr; + break; + } else { + swig_cast_info *tc = SWIG_TypeCheck(to->name,ty); + if (!tc) { + sobj = (SwigPyObject *)sobj->next; + } else { + if (ptr) { + int newmemory = 0; + *ptr = SWIG_TypeCast(tc,vptr,&newmemory); + if (newmemory == SWIG_CAST_NEW_MEMORY) { + assert(own); /* badly formed typemap which will lead to a memory leak - it must set and use own to delete *ptr */ + if (own) + *own = *own | SWIG_CAST_NEW_MEMORY; + } + } + break; + } + } + } else { + if (ptr) *ptr = vptr; + break; + } + } + if (sobj) { + if (((flags & SWIG_POINTER_RELEASE) == SWIG_POINTER_RELEASE) && !sobj->own) { + res = SWIG_ERROR_RELEASE_NOT_OWNED; + } else { + if (own) + *own = *own | sobj->own; + if (flags & SWIG_POINTER_DISOWN) { + sobj->own = 0; + } + if (flags & SWIG_POINTER_CLEAR) { + sobj->ptr = 0; + } + res = SWIG_OK; + } + } else { + if (implicit_conv) { + SwigPyClientData *data = ty ? (SwigPyClientData *) ty->clientdata : 0; + if (data && !data->implicitconv) { + PyObject *klass = data->klass; + if (klass) { + PyObject *impconv; + data->implicitconv = 1; /* avoid recursion and call 'explicit' constructors*/ + impconv = SWIG_Python_CallFunctor(klass, obj); + data->implicitconv = 0; + if (PyErr_Occurred()) { + PyErr_Clear(); + impconv = 0; + } + if (impconv) { + SwigPyObject *iobj = SWIG_Python_GetSwigThis(impconv); + if (iobj) { + void *vptr; + res = SWIG_Python_ConvertPtrAndOwn((PyObject*)iobj, &vptr, ty, 0, 0); + if (SWIG_IsOK(res)) { + if (ptr) { + *ptr = vptr; + /* transfer the ownership to 'ptr' */ + iobj->own = 0; + res = SWIG_AddCast(res); + res = SWIG_AddNewMask(res); + } else { + res = SWIG_AddCast(res); + } + } + } + SWIG_Py_DECREF(impconv); + } + } + } + if (!SWIG_IsOK(res) && obj == Py_None) { + if (ptr) + *ptr = 0; + if (PyErr_Occurred()) + PyErr_Clear(); + res = SWIG_OK; + } + } + } + return res; +} + +/* Convert a function ptr value */ + +SWIGRUNTIME int +SWIG_Python_ConvertFunctionPtr(PyObject *obj, void **ptr, swig_type_info *ty) { + if (!PyCFunction_Check(obj)) { + return SWIG_ConvertPtr(obj, ptr, ty, 0); + } else { + void *vptr = 0; + swig_cast_info *tc; + + /* here we get the method pointer for callbacks */ +#ifndef Py_LIMITED_API + const char *doc = (((PyCFunctionObject *)obj) -> m_ml -> ml_doc); +#else + PyObject* pystr_doc = PyObject_GetAttrString(obj, "__doc__"); + PyObject *bytes = NULL; + const char *doc = pystr_doc ? SWIG_PyUnicode_AsUTF8AndSize(pystr_doc, NULL, &bytes) : 0; +#endif + const char *desc = doc ? strstr(doc, "swig_ptr: ") : 0; + if (desc) + desc = ty ? SWIG_UnpackVoidPtr(desc + 10, &vptr, ty->name) : 0; +#ifdef Py_LIMITED_API + SWIG_Py_XDECREF(bytes); + SWIG_Py_XDECREF(pystr_doc); +#endif + if (!desc) + return SWIG_ERROR; + tc = SWIG_TypeCheck(desc,ty); + if (tc) { + int newmemory = 0; + *ptr = SWIG_TypeCast(tc,vptr,&newmemory); + assert(!newmemory); /* newmemory handling not yet implemented */ + } else { + return SWIG_ERROR; + } + return SWIG_OK; + } +} + +/* Convert a packed pointer value */ + +SWIGRUNTIME int +SWIG_Python_ConvertPacked(PyObject *obj, void *ptr, size_t sz, swig_type_info *ty) { + swig_type_info *to = SwigPyPacked_UnpackData(obj, ptr, sz); + if (!to) return SWIG_ERROR; + if (ty) { + if (to != ty) { + /* check type cast? */ + swig_cast_info *tc = SWIG_TypeCheck(to->name,ty); + if (!tc) return SWIG_ERROR; + } + } + return SWIG_OK; +} + +/* ----------------------------------------------------------------------------- + * Create a new pointer object + * ----------------------------------------------------------------------------- */ + +/* + Create a new instance object, without calling __init__, and set the + 'this' attribute. +*/ + +SWIGRUNTIME PyObject* +SWIG_Python_NewShadowInstance(SwigPyClientData *data, PyObject *swig_this) +{ + PyObject *inst = 0; + PyObject *newraw = data->newraw; + if (newraw) { + inst = PyObject_Call(newraw, data->newargs, NULL); + if (inst) { +#if !defined(SWIG_PYTHON_SLOW_GETSET_THIS) + PyObject **dictptr = _PyObject_GetDictPtr(inst); + if (dictptr != NULL) { + PyObject *dict = *dictptr; + if (dict == NULL) { + dict = PyDict_New(); + *dictptr = dict; + } + if (dict) { + PyDict_SetItem(dict, SWIG_This(), swig_this); + } else{ + SWIG_Py_DECREF(inst); + inst = 0; + } + } +#else + if (PyObject_SetAttr(inst, SWIG_This(), swig_this) == -1) { + SWIG_Py_DECREF(inst); + inst = 0; + } +#endif + } + } else { +#if PY_VERSION_HEX >= 0x03000000 + PyObject *empty_args = PyTuple_New(0); + if (empty_args) { + PyObject *empty_kwargs = PyDict_New(); + if (empty_kwargs) { +#ifndef Py_LIMITED_API + newfunc newfn = ((PyTypeObject *)data->newargs)->tp_new; +#else + newfunc newfn = (newfunc)PyType_GetSlot((PyTypeObject *)data->newargs, Py_tp_new); +#endif + inst = newfn((PyTypeObject *)data->newargs, empty_args, empty_kwargs); + SWIG_Py_DECREF(empty_kwargs); + if (inst) { + if (PyObject_SetAttr(inst, SWIG_This(), swig_this) == -1) { + SWIG_Py_DECREF(inst); + inst = 0; + } else { + PyType_Modified(Py_TYPE(inst)); + } + } + } + SWIG_Py_DECREF(empty_args); + } +#else + PyObject *dict = PyDict_New(); + if (dict) { + PyDict_SetItem(dict, SWIG_This(), swig_this); + inst = PyInstance_NewRaw(data->newargs, dict); + SWIG_Py_DECREF(dict); + } +#endif + } + return inst; +} + +SWIGRUNTIME int +SWIG_Python_SetSwigThis(PyObject *inst, PyObject *swig_this) +{ +#if !defined(SWIG_PYTHON_SLOW_GETSET_THIS) + PyObject **dictptr = _PyObject_GetDictPtr(inst); + if (dictptr != NULL) { + PyObject *dict = *dictptr; + if (dict == NULL) { + dict = PyDict_New(); + *dictptr = dict; + } + if (dict) { + return PyDict_SetItem(dict, SWIG_This(), swig_this); + } else{ + return -1; + } + } +#endif + return PyObject_SetAttr(inst, SWIG_This(), swig_this); +} + + +SWIGINTERN PyObject * +SWIG_Python_InitShadowInstance(PyObject *args) { + PyObject *obj[2]; + if (!SWIG_Python_UnpackTuple(args, "swiginit", 2, 2, obj)) { + return NULL; + } else { + SwigPyObject *sthis = SWIG_Python_GetSwigThis(obj[0]); + if (sthis) { + SWIG_Py_DECREF(SwigPyObject_append((PyObject*) sthis, obj[1])); + } else { + if (SWIG_Python_SetSwigThis(obj[0], obj[1]) != 0) + return NULL; + } + return SWIG_Py_Void(); + } +} + +/* Create a new pointer object */ + +SWIGRUNTIME PyObject * +SWIG_Python_NewPointerObj(PyObject *self, void *ptr, swig_type_info *type, int flags) { + SwigPyClientData *clientdata; + PyObject * robj; + int own; + + if (!ptr) + return SWIG_Py_Void(); + + clientdata = type ? (SwigPyClientData *)(type->clientdata) : 0; + own = (flags & SWIG_POINTER_OWN) ? SWIG_POINTER_OWN : 0; + if (clientdata && clientdata->pytype) { + SwigPyObject *newobj; + if (flags & SWIG_BUILTIN_TP_INIT) { + newobj = (SwigPyObject*) self; + if (newobj->ptr) { +#ifndef Py_LIMITED_API + allocfunc alloc = clientdata->pytype->tp_alloc; +#else + allocfunc alloc = (allocfunc)PyType_GetSlot(clientdata->pytype, Py_tp_alloc); +#endif + PyObject *next_self = alloc(clientdata->pytype, 0); + while (newobj->next) + newobj = (SwigPyObject *) newobj->next; + newobj->next = next_self; + newobj = (SwigPyObject *)next_self; +#ifdef SWIGPYTHON_BUILTIN + newobj->dict = 0; +#endif + } + } else { + newobj = PyObject_New(SwigPyObject, clientdata->pytype); +#ifdef SWIGPYTHON_BUILTIN + if (newobj) { + newobj->dict = 0; + } +#endif + } + if (newobj) { + newobj->ptr = ptr; + newobj->ty = type; + newobj->own = own; + newobj->next = 0; + return (PyObject*) newobj; + } + return SWIG_Py_Void(); + } + + assert(!(flags & SWIG_BUILTIN_TP_INIT)); + + robj = SwigPyObject_New(ptr, type, own); + if (robj && clientdata && !(flags & SWIG_POINTER_NOSHADOW)) { + PyObject *inst = SWIG_Python_NewShadowInstance(clientdata, robj); + SWIG_Py_DECREF(robj); + robj = inst; + } + return robj; +} + +/* Create a new packed object */ + +SWIGRUNTIMEINLINE PyObject * +SWIG_Python_NewPackedObj(void *ptr, size_t sz, swig_type_info *type) { + return ptr ? SwigPyPacked_New((void *) ptr, sz, type) : SWIG_Py_Void(); +} + +/* -----------------------------------------------------------------------------* + * Get type list + * -----------------------------------------------------------------------------*/ + +#ifdef SWIG_LINK_RUNTIME +void *SWIG_ReturnGlobalTypeList(void *); +#endif + +static PyObject *Swig_TypeCache_global = NULL; + +/* The python cached type query */ +SWIGRUNTIME PyObject * +SWIG_Python_TypeCache(void) { + if (Swig_TypeCache_global == NULL) { + Swig_TypeCache_global = PyDict_New(); + } + return Swig_TypeCache_global; +} + +SWIGRUNTIME swig_module_info * +SWIG_Python_GetModule(void *SWIGUNUSEDPARM(clientdata)) { +#ifdef SWIG_LINK_RUNTIME + static void *type_pointer = (void *)0; + /* first check if module already created */ + if (!type_pointer) { + type_pointer = SWIG_ReturnGlobalTypeList((void *)0); + } +#else + void *type_pointer = PyCapsule_Import(SWIGPY_CAPSULE_NAME, 0); + if (PyErr_Occurred()) { + PyErr_Clear(); + type_pointer = (void *)0; + } +#endif + return (swig_module_info *) type_pointer; +} + + +static int interpreter_counter = 0; /* how many (sub-)interpreters are using swig_module's types */ + +SWIGRUNTIME void +SWIG_Python_DestroyModule(PyObject *obj) +{ + swig_module_info *swig_module = (swig_module_info *) PyCapsule_GetPointer(obj, SWIGPY_CAPSULE_NAME); + swig_type_info **types = swig_module->types; + size_t i; + if (--interpreter_counter != 0) /* another sub-interpreter may still be using the swig_module's types */ + return; + for (i =0; i < swig_module->size; ++i) { + swig_type_info *ty = types[i]; + if (ty->owndata) { + SwigPyClientData *data = (SwigPyClientData *) ty->clientdata; + ty->clientdata = 0; + if (data) SwigPyClientData_Del(data); + } + } + SWIG_Py_DECREF(SWIG_This()); + Swig_This_global = NULL; + SWIG_Py_DECREF(SWIG_globals()); + Swig_Globals_global = NULL; + SWIG_Py_DECREF(SWIG_Python_TypeCache()); + Swig_TypeCache_global = NULL; + Swig_Capsule_global = NULL; +} + +SWIGRUNTIME void +SWIG_Python_SetModule(swig_module_info *swig_module) { +#if PY_VERSION_HEX >= 0x03000000 + /* Add a dummy module object into sys.modules */ + PyObject *module = PyImport_AddModule("swig_runtime_data" SWIG_RUNTIME_VERSION); +#else + static PyMethodDef swig_empty_runtime_method_table[] = { {NULL, NULL, 0, NULL} }; /* Sentinel */ + PyObject *module = Py_InitModule("swig_runtime_data" SWIG_RUNTIME_VERSION, swig_empty_runtime_method_table); +#endif + PyObject *pointer = PyCapsule_New((void *) swig_module, SWIGPY_CAPSULE_NAME, SWIG_Python_DestroyModule); + if (pointer && module) { + if (PyModule_AddObject(module, SWIGPY_CAPSULE_ATTR_NAME, pointer) == 0) { + ++interpreter_counter; + Swig_Capsule_global = pointer; + } else { + SWIG_Py_DECREF(pointer); + } + } else { + SWIG_Py_XDECREF(pointer); + } +} + +SWIGRUNTIME swig_type_info * +SWIG_Python_TypeQuery(const char *type) +{ + PyObject *cache = SWIG_Python_TypeCache(); + PyObject *key = SWIG_Python_str_FromChar(type); + PyObject *obj = PyDict_GetItem(cache, key); + swig_type_info *descriptor; + if (obj) { + descriptor = (swig_type_info *) PyCapsule_GetPointer(obj, NULL); + } else { + swig_module_info *swig_module = SWIG_GetModule(0); + descriptor = SWIG_TypeQueryModule(swig_module, swig_module, type); + if (descriptor) { + obj = PyCapsule_New((void*) descriptor, NULL, NULL); + if (obj) { + PyDict_SetItem(cache, key, obj); + SWIG_Py_DECREF(obj); + } + } + } + SWIG_Py_DECREF(key); + return descriptor; +} + +/* + For backward compatibility only +*/ +#define SWIG_POINTER_EXCEPTION 0 +#define SWIG_arg_fail(arg) SWIG_Python_ArgFail(arg) +#define SWIG_MustGetPtr(p, type, argnum, flags) SWIG_Python_MustGetPtr(p, type, argnum, flags) + +SWIGRUNTIME int +SWIG_Python_AddErrMesg(const char* mesg, int infront) +{ + if (PyErr_Occurred()) { + PyObject *type = 0; + PyObject *value = 0; + PyObject *traceback = 0; + PyErr_Fetch(&type, &value, &traceback); + if (value) { + PyObject *old_str = PyObject_Str(value); + PyObject *bytes = NULL; + const char *tmp = SWIG_PyUnicode_AsUTF8AndSize(old_str, NULL, &bytes); + const char *errmesg = tmp ? tmp : "Invalid error message"; + SWIG_Py_XINCREF(type); + PyErr_Clear(); + if (infront) { + PyErr_Format(type, "%s %s", mesg, errmesg); + } else { + PyErr_Format(type, "%s %s", errmesg, mesg); + } + SWIG_Py_XDECREF(bytes); + SWIG_Py_DECREF(old_str); + } + return 1; + } else { + return 0; + } +} + +SWIGRUNTIME int +SWIG_Python_ArgFail(int argnum) +{ + if (PyErr_Occurred()) { + /* add information about failing argument */ + char mesg[256]; + PyOS_snprintf(mesg, sizeof(mesg), "argument number %d:", argnum); + return SWIG_Python_AddErrMesg(mesg, 1); + } else { + return 0; + } +} + +SWIGRUNTIMEINLINE const char * +SwigPyObject_GetDesc(PyObject *self) +{ + SwigPyObject *v = (SwigPyObject *)self; + swig_type_info *ty = v ? v->ty : 0; + return ty ? ty->str : ""; +} + +SWIGRUNTIME void +SWIG_Python_TypeError(const char *type, PyObject *obj) +{ + (void) obj; + if (type) { +#if defined(SWIG_COBJECT_TYPES) + if (obj && SwigPyObject_Check(obj)) { + const char *otype = (const char *) SwigPyObject_GetDesc(obj); + if (otype) { + PyErr_Format(PyExc_TypeError, "a '%s' is expected, 'SwigPyObject(%s)' is received", + type, otype); + return; + } + } else +#endif + { +#ifndef Py_LIMITED_API + /* tp_name is not accessible */ + const char *otype = (obj ? obj->ob_type->tp_name : 0); + if (otype) { + PyObject *str = PyObject_Str(obj); + PyObject *bytes = NULL; + const char *cstr = str ? SWIG_PyUnicode_AsUTF8AndSize(str, NULL, &bytes) : 0; + if (cstr) { + PyErr_Format(PyExc_TypeError, "a '%s' is expected, '%s(%s)' is received", + type, otype, cstr); + } else { + PyErr_Format(PyExc_TypeError, "a '%s' is expected, '%s' is received", + type, otype); + } + SWIG_Py_XDECREF(bytes); + SWIG_Py_XDECREF(str); + return; + } +#endif + } + PyErr_Format(PyExc_TypeError, "a '%s' is expected", type); + } else { + PyErr_Format(PyExc_TypeError, "unexpected type is received"); + } +} + + +/* Convert a pointer value, signal an exception on a type mismatch */ +SWIGRUNTIME void * +SWIG_Python_MustGetPtr(PyObject *obj, swig_type_info *ty, int SWIGUNUSEDPARM(argnum), int flags) { + void *result; + if (SWIG_Python_ConvertPtr(obj, &result, ty, flags) == -1) { + PyErr_Clear(); + } + return result; +} + +#ifdef SWIGPYTHON_BUILTIN +SWIGRUNTIME int +SWIG_Python_NonDynamicSetAttr(PyObject *obj, PyObject *name, PyObject *value) { + PyTypeObject *tp = obj->ob_type; + PyObject *descr; + PyObject *encoded_name; + descrsetfunc f; + int res = -1; + +# ifdef Py_USING_UNICODE + if (PyString_Check(name)) { + name = PyUnicode_Decode(PyString_AsString(name), PyString_Size(name), NULL, NULL); + if (!name) + return -1; + } else if (!PyUnicode_Check(name)) +# else + if (!PyString_Check(name)) +# endif + { + PyErr_Format(PyExc_TypeError, "attribute name must be string, not '%.200s'", name->ob_type->tp_name); + return -1; + } else { + SWIG_Py_INCREF(name); + } + + if (!tp->tp_dict) { + if (PyType_Ready(tp) != 0) + goto done; + } + + descr = _PyType_Lookup(tp, name); + f = NULL; + if (descr != NULL) + f = descr->ob_type->tp_descr_set; + if (!f) { + if (PyString_Check(name)) { + encoded_name = name; + SWIG_Py_INCREF(name); + } else { + encoded_name = PyUnicode_AsUTF8String(name); + if (!encoded_name) + goto done; + } + PyErr_Format(PyExc_AttributeError, "'%.100s' object has no attribute '%.200s'", tp->tp_name, PyString_AsString(encoded_name)); + SWIG_Py_DECREF(encoded_name); + } else { + res = f(descr, obj, value); + } + + done: + SWIG_Py_DECREF(name); + return res; +} +#endif + + +#ifdef __cplusplus +} +#endif + + + +#define SWIG_exception_fail(code, msg) do { SWIG_Error(code, msg); SWIG_fail; } while(0) + +#define SWIG_contract_assert(expr, msg) do { if (!(expr)) { SWIG_Error(SWIG_RuntimeError, msg); SWIG_fail; } } while (0) + + + + #define SWIG_exception(code, msg) do { SWIG_Error(code, msg); SWIG_fail;; } while(0) + + +/* -------- TYPES TABLE (BEGIN) -------- */ + +#define SWIGTYPE_p_char swig_types[0] +#define SWIGTYPE_p_float swig_types[1] +#define SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText swig_types[2] +#define SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText swig_types[3] +#define SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece swig_types[4] +#define SWIGTYPE_p_sentencepiece__SentenceIterator swig_types[5] +#define SWIGTYPE_p_sentencepiece__SentencePieceNormalizer swig_types[6] +#define SWIGTYPE_p_sentencepiece__SentencePieceProcessor swig_types[7] +#define SWIGTYPE_p_sentencepiece__SentencePieceTrainer swig_types[8] +#define SWIGTYPE_p_std__string swig_types[9] +#define SWIGTYPE_p_std__unordered_mapT_std__string_std__string_t swig_types[10] +#define SWIGTYPE_p_std__vectorT_absl__string_view_t swig_types[11] +#define SWIGTYPE_p_std__vectorT_int_t swig_types[12] +#define SWIGTYPE_p_std__vectorT_std__vectorT_absl__string_view_t_t swig_types[13] +#define SWIGTYPE_p_std__vectorT_std__vectorT_int_t_t swig_types[14] +static swig_type_info *swig_types[16]; +static swig_module_info swig_module = {swig_types, 15, 0, 0, 0, 0}; +#define SWIG_TypeQuery(name) SWIG_TypeQueryModule(&swig_module, &swig_module, name) +#define SWIG_MangledTypeQuery(name) SWIG_MangledTypeQueryModule(&swig_module, &swig_module, name) + +/* -------- TYPES TABLE (END) -------- */ + +#ifdef SWIG_TypeQuery +# undef SWIG_TypeQuery +#endif +#define SWIG_TypeQuery SWIG_Python_TypeQuery + +/*----------------------------------------------- + @(target):= _sentencepiece.so + ------------------------------------------------*/ +#if PY_VERSION_HEX >= 0x03000000 +# define SWIG_init PyInit__sentencepiece + +#else +# define SWIG_init init_sentencepiece + +#endif + +#ifdef __cplusplus +#include +/* SwigValueWrapper is described in swig.swg */ +template class SwigValueWrapper { + struct SwigSmartPointer { + T *ptr; + SwigSmartPointer(T *p) : ptr(p) { } + ~SwigSmartPointer() { delete ptr; } + SwigSmartPointer& operator=(SwigSmartPointer& rhs) { T* oldptr = ptr; ptr = 0; delete oldptr; ptr = rhs.ptr; rhs.ptr = 0; return *this; } + void reset(T *p) { T* oldptr = ptr; ptr = 0; delete oldptr; ptr = p; } + } pointer; + SwigValueWrapper& operator=(const SwigValueWrapper& rhs); + SwigValueWrapper(const SwigValueWrapper& rhs); +public: + SwigValueWrapper() : pointer(0) { } + SwigValueWrapper& operator=(const T& t) { SwigSmartPointer tmp(new T(t)); pointer = tmp; return *this; } +#if __cplusplus >=201103L + SwigValueWrapper& operator=(T&& t) { SwigSmartPointer tmp(new T(std::move(t))); pointer = tmp; return *this; } + operator T&&() const { return std::move(*pointer.ptr); } +#else + operator T&() const { return *pointer.ptr; } +#endif + T *operator&() const { return pointer.ptr; } + static void reset(SwigValueWrapper& t, T *p) { t.pointer.reset(p); } +}; + +/* + * SwigValueInit() is a generic initialisation solution as the following approach: + * + * T c_result = T(); + * + * doesn't compile for all types for example: + * + * unsigned int c_result = unsigned int(); + */ +template T SwigValueInit() { + return T(); +} + +#if __cplusplus >=201103L +# define SWIG_STD_MOVE(OBJ) std::move(OBJ) +#else +# define SWIG_STD_MOVE(OBJ) OBJ +#endif + +#endif + + +#define SWIG_as_voidptr(a) const_cast< void * >(static_cast< const void * >(a)) +#define SWIG_as_voidptrptr(a) ((void)SWIG_as_voidptr(*a),reinterpret_cast< void** >(a)) + + +#include + + +namespace swig { + class SwigPtr_PyObject { + protected: + PyObject *_obj; + + public: + SwigPtr_PyObject() :_obj(0) + { + } + + SwigPtr_PyObject(const SwigPtr_PyObject& item) : _obj(item._obj) + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + SWIG_Py_XINCREF(_obj); + SWIG_PYTHON_THREAD_END_BLOCK; + } + + SwigPtr_PyObject(PyObject *obj, bool initial_ref = true) :_obj(obj) + { + if (initial_ref) { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + SWIG_Py_XINCREF(_obj); + SWIG_PYTHON_THREAD_END_BLOCK; + } + } + + SwigPtr_PyObject & operator=(const SwigPtr_PyObject& item) + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + SWIG_Py_XINCREF(item._obj); + SWIG_Py_XDECREF(_obj); + _obj = item._obj; + SWIG_PYTHON_THREAD_END_BLOCK; + return *this; + } + + ~SwigPtr_PyObject() + { + SWIG_PYTHON_THREAD_BEGIN_BLOCK; + SWIG_Py_XDECREF(_obj); + SWIG_PYTHON_THREAD_END_BLOCK; + } + + operator PyObject *() const + { + return _obj; + } + + PyObject *operator->() const + { + return _obj; + } + }; +} + + +namespace swig { + struct SwigVar_PyObject : SwigPtr_PyObject { + SwigVar_PyObject(PyObject* obj = 0) : SwigPtr_PyObject(obj, false) { } + + SwigVar_PyObject & operator = (PyObject* obj) + { + SWIG_Py_XDECREF(_obj); + _obj = obj; + return *this; + } + }; +} + + + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { +PyObject* kUnicodeInput = reinterpret_cast(0x1); +PyObject* kByteInput = reinterpret_cast(0x2); + +using BytesArray = std::vector; + +inline void ReleaseResultObject(PyObject *obj) { + if (obj != nullptr && obj != kUnicodeInput && obj != kByteInput) { + Py_XDECREF(obj); + } +} + +class PyInputString { + public: + explicit PyInputString(PyObject* obj) { + if (PyUnicode_Check(obj)) { + str_ = const_cast(PyUnicode_AsUTF8AndSize(obj, &size_)); + input_type_ = kUnicodeInput; + } else if (PyBytes_Check(obj)) { + PyBytes_AsStringAndSize(obj, &str_, &size_); + input_type_ = kByteInput; + } else { + str_ = nullptr; + } + } + absl::string_view str() const { return absl::string_view(data(), size()); } + const char* data() const { return str_; } + Py_ssize_t size() const { return size_; } + bool IsAvalable() const { return str_ != nullptr; } + PyObject *input_type() const { return input_type_; } + + static bool IsUnicode(PyObject *resultobj) { + return (resultobj == nullptr || resultobj == kUnicodeInput); + } + + private: + PyObject* input_type_ = nullptr; + char* str_ = nullptr; + Py_ssize_t size_ = 0; +}; + +PyObject* MakePyOutputString(const std::string& output, + PyObject *resultobj) { + if (PyInputString::IsUnicode(resultobj)) { + return PyUnicode_FromStringAndSize(output.data(), output.size()); + } + return PyBytes_FromStringAndSize(output.data(), output.size()); +} + +PyObject* MakePyOutputBytes(const sentencepiece::util::bytes& output) { + return PyBytes_FromStringAndSize(output.data(), output.size()); +} + +int ToSwigError(sentencepiece::util::StatusCode code) { + switch (code) { + case sentencepiece::util::StatusCode::kNotFound: + return SWIG_IOError; + case sentencepiece::util::StatusCode::kOutOfRange: + return SWIG_IndexError; + case sentencepiece::util::StatusCode::kInvalidArgument: + return SWIG_SyntaxError; + default: + return SWIG_RuntimeError; + } + return SWIG_RuntimeError; +} + +class PySentenceIterator : public sentencepiece::SentenceIterator { + public: + PySentenceIterator(PyObject *iter) : iter_(iter) { + item_ = PyIter_Next(iter_); + CopyValue(); + } + + ~PySentenceIterator() { + // Py_XDECREF(iter_); + } + + bool done() const override { + return item_ == nullptr; + } + + void Next() override { + item_ = PyIter_Next(iter_); + CopyValue(); + } + + const std::string &value() const override { + return value_; + } + + sentencepiece::util::Status status() const override { + return status_; + } + + private: + void CopyValue() { + if (item_ == nullptr) return; + const PyInputString ustring(item_); + if (ustring.IsAvalable()) { + const char *data = ustring.data(); + size_t size = ustring.size(); + while (size > 0) { + if (data[size - 1] == '\r' || data[size - 1] == '\n') + --size; + else + break; + } + value_.assign(data, size); + } else { + status_ = sentencepiece::util::Status(sentencepiece::util::StatusCode::kInternal, + "Not a string."); + } + Py_XDECREF(item_); + } + PyObject *iter_ = nullptr; + PyObject *item_ = nullptr; + std::string value_; + sentencepiece::util::Status status_; +}; + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + std::vector *ids, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (!add_bos && !add_eos && !reverse) return; + if (reverse) std::reverse(ids->begin(), ids->end()); + if (add_bos) ids->insert(ids->begin(), sp.bos_id()); + if (add_eos) ids->push_back(sp.eos_id()); +} + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + std::vector *pieces, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (!add_bos && !add_eos && !reverse && !emit_unk_piece) return; + if (reverse) std::reverse(pieces->begin(), pieces->end()); + if (add_bos) pieces->insert(pieces->begin(), sp.IdToPiece(sp.bos_id())); + if (add_eos) pieces->push_back(sp.IdToPiece(sp.eos_id())); + if (emit_unk_piece) { + const auto &unk = sp.IdToPiece(sp.unk_id()); + for (auto &piece : *pieces) { + const int id = sp.PieceToId(piece); + if (id == sp.unk_id()) { + piece = unk; + } + } + } +} + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + sentencepiece::util::bytes *proto, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (add_bos || add_eos || reverse || emit_unk_piece) { + throw sentencepiece::util::Status( + sentencepiece::util::StatusCode::kUnimplemented, + "add_bos, add_eos, reverse, and emit_unk_piece is not supported in proto API"); + } +} + +inline void RewriteIds(const sentencepiece::SentencePieceProcessor &sp, + sentencepiece::ImmutableSentencePieceText *proto, + bool add_bos, bool add_eos, bool reverse, bool emit_unk_piece) { + if (add_bos || add_eos || reverse || emit_unk_piece) { + throw sentencepiece::util::Status( + sentencepiece::util::StatusCode::kUnimplemented, + "add_bos, add_eos, reverse, and emit_unk_piece is not supported in proto API"); + } +} + +inline void CheckIds(const std::vector &ids, int num_pieces) { + for (int id : ids) { + if (id < 0 || id >= num_pieces) { + throw sentencepiece::util::Status( + sentencepiece::util::StatusCode::kOutOfRange, + "piece id is out of range."); + } + } +} + +inline void CheckIds(const std::vector &ids, int num_pieces) {} + +inline void CheckIdsBatch(const std::vector> &ids, int num_pieces) { + for (const auto &v : ids) CheckIds(v, num_pieces); +} + +template +inline void ConvertToUnicodeSpans(T *proto) {} + +template <> +inline void ConvertToUnicodeSpans(sentencepiece::ImmutableSentencePieceText *proto) { + proto->ConvertToUnicodeSpans(); +} + +template <> +inline void ConvertToUnicodeSpans(sentencepiece::ImmutableNBestSentencePieceText *proto) { + proto->ConvertToUnicodeSpans(); +} + +class ThreadPool { + public: + explicit ThreadPool(size_t request_size) : + request_size_(request_size) {} + + virtual ~ThreadPool() { + for (auto &task : tasks_) { + task.join(); + } + } + + void Schedule(std::function closure) { + static constexpr size_t kMinThreadSize = 2; + if (request_size_ < kMinThreadSize) { + closure(); + } else { + tasks_.emplace_back(closure); + } + } + + private: + size_t request_size_ = 0; + std::vector tasks_; +}; + +template +inline void InitNumThreads(const std::vector &ins, int *num_threads) { + if (*num_threads < 0) { + *num_threads = std::thread::hardware_concurrency(); + } + *num_threads = std::max(1, + std::min({*num_threads, + static_cast(ins.size()), 256})); +} + +#define DEFINE_ENCODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \ + std::vector outs(ins.size()); \ + InitNumThreads(ins, &num_threads); \ + { \ + ThreadPool pool(ins.size()); \ + std::atomic index = 0; \ + for (int n = 0; n < num_threads; ++n) { \ + pool.Schedule([&]() { \ + size_t i = 0; \ + while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { \ + auto out = enable_sampling ? \ + self->Sample##FuncName(ins[i], \ + nbest_size, alpha) : \ + self->FuncName(ins[i]); \ + RewriteIds(*self, &out, add_bos, add_eos, reverse, \ + emit_unk_piece); \ + ConvertToUnicodeSpans(&out); \ + outs[i] = std::move(out); \ + } \ + }); \ + } \ + } \ + return outs; + +#define DEFINE_DECODE_BATCH_FUNC_IMPL(FuncName, InType, OutType) \ + std::vector outs(ins.size()); \ + InitNumThreads(ins, &num_threads); \ + { \ + std::atomic index = 0; \ + ThreadPool pool(ins.size()); \ + for (int n = 0; n < num_threads; ++n) { \ + pool.Schedule([&]() { \ + size_t i = 0; \ + while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { \ + auto out = self->FuncName(ins[i]); \ + ConvertToUnicodeSpans(&out); \ + outs[i] = std::move(out); \ + } \ + }); \ + } \ + } \ + return outs; + +} // namespace + + +SWIGINTERNINLINE PyObject* + SWIG_From_unsigned_SS_int (unsigned int value) +{ + return PyInt_FromSize_t((size_t) value); +} + +SWIGINTERN sentencepiece::util::bytes const &sentencepiece_ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes(sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *self){ + return self->surface(); + } +SWIGINTERN sentencepiece::util::bytes const &sentencepiece_ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes(sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *self){ + return self->piece(); + } + + #define SWIG_From_long PyInt_FromLong + + +SWIGINTERNINLINE PyObject* +SWIG_From_unsigned_SS_long (unsigned long value) +{ + return (value > LONG_MAX) ? + PyLong_FromUnsignedLong(value) : PyInt_FromLong(static_cast< long >(value)); +} + + +#include +#if !defined(SWIG_NO_LLONG_MAX) +# if !defined(LLONG_MAX) && defined(__GNUC__) && defined (__LONG_LONG_MAX__) +# define LLONG_MAX __LONG_LONG_MAX__ +# define LLONG_MIN (-LLONG_MAX - 1LL) +# define ULLONG_MAX (LLONG_MAX * 2ULL + 1ULL) +# endif +#endif + + +#if defined(LLONG_MAX) && !defined(SWIG_LONG_LONG_AVAILABLE) +# define SWIG_LONG_LONG_AVAILABLE +#endif + + +#ifdef SWIG_LONG_LONG_AVAILABLE +SWIGINTERNINLINE PyObject* +SWIG_From_unsigned_SS_long_SS_long (unsigned long long value) +{ + return (value > LONG_MAX) ? + PyLong_FromUnsignedLongLong(value) : PyInt_FromLong(static_cast< long >(value)); +} +#endif + + +SWIGINTERNINLINE PyObject * +SWIG_From_size_t (size_t value) +{ +#ifdef SWIG_LONG_LONG_AVAILABLE + if (sizeof(size_t) <= sizeof(unsigned long)) { +#endif + return SWIG_From_unsigned_SS_long (static_cast< unsigned long >(value)); +#ifdef SWIG_LONG_LONG_AVAILABLE + } else { + /* assume sizeof(size_t) <= sizeof(unsigned long long) */ + return SWIG_From_unsigned_SS_long_SS_long (static_cast< unsigned long long >(value)); + } +#endif +} + + +SWIGINTERN int +SWIG_AsVal_double (PyObject *obj, double *val) +{ + int res = SWIG_TypeError; + if (PyFloat_Check(obj)) { + if (val) *val = PyFloat_AsDouble(obj); + return SWIG_OK; +#if PY_VERSION_HEX < 0x03000000 + } else if (PyInt_Check(obj)) { + if (val) *val = (double) PyInt_AsLong(obj); + return SWIG_OK; +#endif + } else if (PyLong_Check(obj)) { + double v = PyLong_AsDouble(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + int dispatch = 0; + double d = PyFloat_AsDouble(obj); + if (!PyErr_Occurred()) { + if (val) *val = d; + return SWIG_AddCast(SWIG_OK); + } else { + PyErr_Clear(); + } + if (!dispatch) { + long v = PyLong_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_AddCast(SWIG_AddCast(SWIG_OK)); + } else { + PyErr_Clear(); + } + } + } +#endif + return res; +} + + +#include + + +#include + + +SWIGINTERNINLINE int +SWIG_CanCastAsInteger(double *d, double min, double max) { + double x = *d; + if ((min <= x && x <= max)) { + double fx, cx, rd; + errno = 0; + fx = floor(x); + cx = ceil(x); + rd = ((x - fx) < 0.5) ? fx : cx; /* simple rint */ + if ((errno == EDOM) || (errno == ERANGE)) { + errno = 0; + } else { + double summ, reps, diff; + if (rd < x) { + diff = x - rd; + } else if (rd > x) { + diff = rd - x; + } else { + return 1; + } + summ = rd + x; + reps = diff/summ; + if (reps < 8*DBL_EPSILON) { + *d = rd; + return 1; + } + } + } + return 0; +} + + +SWIGINTERN int +SWIG_AsVal_long (PyObject *obj, long* val) +{ +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(obj)) { + if (val) *val = PyInt_AsLong(obj); + return SWIG_OK; + } else +#endif + if (PyLong_Check(obj)) { + long v = PyLong_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + return SWIG_OverflowError; + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + int dispatch = 0; + long v = PyInt_AsLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_AddCast(SWIG_OK); + } else { + PyErr_Clear(); + } + if (!dispatch) { + double d; + int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); + // Largest double not larger than LONG_MAX (not portably calculated easily) + // Note that double(LONG_MAX) is stored in a double rounded up by one (for 64-bit long) + // 0x7ffffffffffffc00LL == (int64_t)std::nextafter(double(__uint128_t(LONG_MAX)+1), double(0)) + const double long_max = sizeof(long) == 8 ? 0x7ffffffffffffc00LL : LONG_MAX; + // No equivalent needed for 64-bit double(LONG_MIN) is exactly LONG_MIN + if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, LONG_MIN, long_max)) { + if (val) *val = (long)(d); + return res; + } + } + } +#endif + return SWIG_TypeError; +} + + +SWIGINTERN int +SWIG_AsVal_int (PyObject * obj, int *val) +{ + long v; + int res = SWIG_AsVal_long (obj, &v); + if (SWIG_IsOK(res)) { + if ((v < INT_MIN || v > INT_MAX)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< int >(v); + } + } + return res; +} + + + #define SWIG_From_double PyFloat_FromDouble + + +SWIGINTERNINLINE PyObject * +SWIG_From_float (float value) +{ + return SWIG_From_double (value); +} + +SWIGINTERN sentencepiece::util::bytes const &sentencepiece_ImmutableSentencePieceText__text_as_bytes(sentencepiece::ImmutableSentencePieceText const *self){ + return self->text(); + } + +SWIGINTERN swig_type_info* +SWIG_pchar_descriptor(void) +{ + static swig_type_info* info = 0; + if (!info) { + info = SWIG_TypeQuery("_p_char"); + } + return info; +} + + +/* Return string from Python obj. NOTE: obj must remain in scope in order + to use the returned cptr (but only when alloc is set to SWIG_OLDOBJ) */ +SWIGINTERN int +SWIG_AsCharPtrAndSize(PyObject *obj, char **cptr, size_t *psize, int *alloc) +{ +#if PY_VERSION_HEX>=0x03000000 +#if defined(SWIG_PYTHON_STRICT_BYTE_CHAR) + if (PyBytes_Check(obj)) +#else + if (PyUnicode_Check(obj)) +#endif +#else + if (PyString_Check(obj)) +#endif + { + char *cstr; Py_ssize_t len; + PyObject *bytes = NULL; + int ret = SWIG_OK; + if (alloc) + *alloc = SWIG_OLDOBJ; +#if PY_VERSION_HEX>=0x03000000 && defined(SWIG_PYTHON_STRICT_BYTE_CHAR) + if (PyBytes_AsStringAndSize(obj, &cstr, &len) == -1) + return SWIG_TypeError; +#else + cstr = (char *)SWIG_PyUnicode_AsUTF8AndSize(obj, &len, &bytes); + if (!cstr) + return SWIG_TypeError; + /* The returned string is only duplicated if the char * returned is not owned and memory managed by obj */ + if (bytes && cptr) { + if (alloc) { + cstr = reinterpret_cast< char* >(memcpy(new char[len + 1], cstr, sizeof(char)*(len + 1))); + *alloc = SWIG_NEWOBJ; + } else { + /* alloc must be set in order to clean up allocated memory */ + return SWIG_RuntimeError; + } + } +#endif + if (cptr) *cptr = cstr; + if (psize) *psize = len + 1; + SWIG_Py_XDECREF(bytes); + return ret; + } else { +#if defined(SWIG_PYTHON_2_UNICODE) +#if defined(SWIG_PYTHON_STRICT_BYTE_CHAR) +#error "Cannot use both SWIG_PYTHON_2_UNICODE and SWIG_PYTHON_STRICT_BYTE_CHAR at once" +#endif +#if PY_VERSION_HEX<0x03000000 + if (PyUnicode_Check(obj)) { + char *cstr; Py_ssize_t len; + if (!alloc && cptr) { + return SWIG_RuntimeError; + } + obj = PyUnicode_AsUTF8String(obj); + if (!obj) + return SWIG_TypeError; + if (PyString_AsStringAndSize(obj, &cstr, &len) != -1) { + if (cptr) { + if (alloc) *alloc = SWIG_NEWOBJ; + *cptr = reinterpret_cast< char* >(memcpy(new char[len + 1], cstr, sizeof(char)*(len + 1))); + } + if (psize) *psize = len + 1; + + SWIG_Py_XDECREF(obj); + return SWIG_OK; + } else { + SWIG_Py_XDECREF(obj); + } + } +#endif +#endif + + swig_type_info* pchar_descriptor = SWIG_pchar_descriptor(); + if (pchar_descriptor) { + void* vptr = 0; + if (SWIG_ConvertPtr(obj, &vptr, pchar_descriptor, 0) == SWIG_OK) { + if (cptr) *cptr = (char *) vptr; + if (psize) *psize = vptr ? (strlen((char *)vptr) + 1) : 0; + if (alloc) *alloc = SWIG_OLDOBJ; + return SWIG_OK; + } + } + } + return SWIG_TypeError; +} + + + + + +/* Getting isfinite working pre C99 across multiple platforms is non-trivial. Users can provide SWIG_isfinite on older platforms. */ +#ifndef SWIG_isfinite +/* isfinite() is a macro for C99 */ +# if defined(isfinite) +# define SWIG_isfinite(X) (isfinite(X)) +# elif defined(__cplusplus) && __cplusplus >= 201103L +/* Use a template so that this works whether isfinite() is std::isfinite() or + * in the global namespace. The reality seems to vary between compiler + * versions. + * + * Make sure namespace std exists to avoid compiler warnings. + * + * extern "C++" is required as this fragment can end up inside an extern "C" { } block + */ +namespace std { } +extern "C++" template +inline int SWIG_isfinite_func(T x) { + using namespace std; + return isfinite(x); +} +# define SWIG_isfinite(X) (SWIG_isfinite_func(X)) +# elif defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 2)) +# define SWIG_isfinite(X) (__builtin_isfinite(X)) +# elif defined(_MSC_VER) +# define SWIG_isfinite(X) (_finite(X)) +# elif defined(__sun) && defined(__SVR4) +# include +# define SWIG_isfinite(X) (finite(X)) +# endif +#endif + + +/* Accept infinite as a valid float value unless we are unable to check if a value is finite */ +#ifdef SWIG_isfinite +# define SWIG_Float_Overflow_Check(X) ((X < -FLT_MAX || X > FLT_MAX) && SWIG_isfinite(X)) +#else +# define SWIG_Float_Overflow_Check(X) ((X < -FLT_MAX || X > FLT_MAX)) +#endif + + +SWIGINTERN int +SWIG_AsVal_float (PyObject * obj, float *val) +{ + double v; + int res = SWIG_AsVal_double (obj, &v); + if (SWIG_IsOK(res)) { + if (SWIG_Float_Overflow_Check(v)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< float >(v); + } + } + return res; +} + + +SWIGINTERNINLINE PyObject* + SWIG_From_int (int value) +{ + return PyInt_FromLong((long) value); +} + + +SWIGINTERNINLINE PyObject* + SWIG_From_bool (bool value) +{ + return PyBool_FromLong(value ? 1 : 0); +} + +SWIGINTERN sentencepiece::util::Status sentencepiece_SentencePieceProcessor_LoadFromFile(sentencepiece::SentencePieceProcessor *self,absl::string_view arg){ + return self->Load(arg); + } + +SWIGINTERN int +SWIG_AsVal_bool (PyObject *obj, bool *val) +{ + int r; + if (!PyBool_Check(obj)) + return SWIG_ERROR; + r = PyObject_IsTrue(obj); + if (r == -1) + return SWIG_ERROR; + if (val) *val = r ? true : false; + return SWIG_OK; +} + +SWIGINTERN std::vector< int > sentencepiece_SentencePieceProcessor__EncodeAsIds(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto ids = enable_sampling ? + self->SampleEncodeAsIds(text, nbest_size, alpha) : + self->EncodeAsIds(text); + RewriteIds(*self, &ids, add_bos, add_eos, reverse, emit_unk_piece); + return ids; + } +SWIGINTERN std::vector< std::string > sentencepiece_SentencePieceProcessor__EncodeAsPieces(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto pieces = enable_sampling ? + self->SampleEncodeAsPieces(text, nbest_size, alpha) : + self->EncodeAsPieces(text); + RewriteIds(*self, &pieces, add_bos, add_eos, reverse, emit_unk_piece); + return pieces; + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__EncodeAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto proto = enable_sampling ? + self->SampleEncodeAsSerializedProto(text, nbest_size, alpha) : + self->EncodeAsSerializedProto(text); + RewriteIds(*self, &proto, add_bos, add_eos, reverse, emit_unk_piece); + return proto; + } +SWIGINTERN sentencepiece::ImmutableSentencePieceText sentencepiece_SentencePieceProcessor__EncodeAsImmutableProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto proto = enable_sampling ? + self->SampleEncodeAsImmutableProto(text, nbest_size, alpha) : + self->EncodeAsImmutableProto(text); + proto.ConvertToUnicodeSpans(); + RewriteIds(*self, &proto, add_bos, add_eos, reverse, emit_unk_piece); + return proto; + } +SWIGINTERN std::vector< std::vector< int > > sentencepiece_SentencePieceProcessor__EncodeAsIdsBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &ins,int num_threads,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsIds, + absl::string_view, std::vector); + } +SWIGINTERN std::vector< std::vector< std::string > > sentencepiece_SentencePieceProcessor__EncodeAsPiecesBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &ins,int num_threads,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsPieces, + absl::string_view, std::vector); + } +SWIGINTERN BytesArray sentencepiece_SentencePieceProcessor__EncodeAsSerializedProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &ins,int num_threads,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsSerializedProto, + absl::string_view, + sentencepiece::util::bytes); + } +SWIGINTERN std::vector< sentencepiece::ImmutableSentencePieceText > sentencepiece_SentencePieceProcessor__EncodeAsImmutableProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &ins,int num_threads,bool enable_sampling,int nbest_size,float alpha,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + DEFINE_ENCODE_BATCH_FUNC_IMPL(EncodeAsImmutableProto, + absl::string_view, + sentencepiece::ImmutableSentencePieceText); + } +SWIGINTERN std::string sentencepiece_SentencePieceProcessor__DecodeIds(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){ + CheckIds(ids, self->GetPieceSize()); + return self->DecodeIds(ids); + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__DecodeIdsAsBytes(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){ + CheckIds(ids, self->GetPieceSize()); + return self->DecodeIds(ids); + } +SWIGINTERN std::string sentencepiece_SentencePieceProcessor__DecodePieces(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &pieces){ + return self->DecodePieces(pieces); + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){ + CheckIds(ids, self->GetPieceSize()); + return self->DecodeIdsAsSerializedProto(ids); + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &pieces){ + CheckIds(pieces, self->GetPieceSize()); + return self->DecodePiecesAsSerializedProto(pieces); + } +SWIGINTERN sentencepiece::ImmutableSentencePieceText sentencepiece_SentencePieceProcessor__DecodeIdsAsImmutableProto(sentencepiece::SentencePieceProcessor const *self,std::vector< int > const &ids){ + CheckIds(ids, self->GetPieceSize()); + auto proto = self->DecodeIdsAsImmutableProto(ids); + proto.ConvertToUnicodeSpans(); + return proto; + } +SWIGINTERN sentencepiece::ImmutableSentencePieceText sentencepiece_SentencePieceProcessor__DecodePiecesAsImmutableProto(sentencepiece::SentencePieceProcessor const *self,std::vector< absl::string_view > const &pieces){ + CheckIds(pieces, self->GetPieceSize()); + auto proto= self->DecodePiecesAsImmutableProto(pieces); + proto.ConvertToUnicodeSpans(); + return proto; + } +SWIGINTERN std::vector< std::string > sentencepiece_SentencePieceProcessor__DecodeIdsBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< int > > const &ins,int num_threads){ + CheckIdsBatch(ins, self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string); + } +SWIGINTERN BytesArray sentencepiece_SentencePieceProcessor__DecodeIdsAsBytesBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< int > > const &ins,int num_threads){ + CheckIdsBatch(ins, self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIds, int, std::string); + } +SWIGINTERN BytesArray sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< int > > const &ins,int num_threads){ + CheckIdsBatch(ins, self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsSerializedProto, int, + sentencepiece::util::bytes); + } +SWIGINTERN std::vector< sentencepiece::ImmutableSentencePieceText > sentencepiece_SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< int > > const &ins,int num_threads){ + CheckIdsBatch(ins, self->GetPieceSize()); + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodeIdsAsImmutableProto, int, + sentencepiece::ImmutableSentencePieceText); + } +SWIGINTERN std::vector< std::string > sentencepiece_SentencePieceProcessor__DecodePiecesBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< absl::string_view > > const &ins,int num_threads){ + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePieces, std::string, std::string); + } +SWIGINTERN BytesArray sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< absl::string_view > > const &ins,int num_threads){ + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsSerializedProto, std::string, + sentencepiece::util::bytes); + } +SWIGINTERN std::vector< sentencepiece::ImmutableSentencePieceText > sentencepiece_SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch(sentencepiece::SentencePieceProcessor const *self,std::vector< std::vector< absl::string_view > > const &ins,int num_threads){ + DEFINE_DECODE_BATCH_FUNC_IMPL(DecodePiecesAsImmutableProto, std::string, + sentencepiece::ImmutableSentencePieceText); + } +SWIGINTERN std::vector< std::vector< int > > sentencepiece_SentencePieceProcessor__NBestEncodeAsIds(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto idss = self->NBestEncodeAsIds(text, nbest_size); + for (auto &ids : idss) { + RewriteIds(*self, &ids, add_bos, add_eos, reverse, emit_unk_piece); + } + return idss; + } +SWIGINTERN std::vector< std::vector< std::string > > sentencepiece_SentencePieceProcessor__NBestEncodeAsPieces(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto piecess = self->NBestEncodeAsPieces(text, nbest_size); + for (auto &pieces : piecess) { + RewriteIds(*self, &pieces, add_bos, add_eos, reverse, emit_unk_piece); + } + return piecess; + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__NBestEncodeAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + RewriteIds(*self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + return self->NBestEncodeAsSerializedProto(text, nbest_size); + } +SWIGINTERN sentencepiece::ImmutableNBestSentencePieceText sentencepiece_SentencePieceProcessor__NBestEncodeAsImmutableProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int nbest_size,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + RewriteIds(*self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + auto proto = self->NBestEncodeAsImmutableProto(text, nbest_size); + proto.ConvertToUnicodeSpans(); + return proto; + } +SWIGINTERN std::vector< std::pair< std::vector< int >,float > > sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsIds(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int num_samples,float alpha,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto idss = self->SampleEncodeAndScoreAsIds(text, num_samples, + alpha, wor, include_best); + for (auto &ids : idss) { + RewriteIds(*self, &ids.first, add_bos, add_eos, reverse, emit_unk_piece); + } + return idss; + } +SWIGINTERN std::vector< std::pair< std::vector< std::string >,float > > sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsPieces(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int num_samples,float alpha,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + auto piecess = self->SampleEncodeAndScoreAsPieces(text, num_samples, + alpha, wor, include_best); + for (auto &pieces : piecess) { + RewriteIds(*self, &pieces.first, add_bos, add_eos, reverse, emit_unk_piece); + } + return piecess; + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int num_samples,float alpha,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + RewriteIds(*self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + return self->SampleEncodeAndScoreAsSerializedProto(text, num_samples, + alpha, wor, include_best); + } +SWIGINTERN sentencepiece::ImmutableNBestSentencePieceText sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto(sentencepiece::SentencePieceProcessor const *self,absl::string_view text,int num_samples,float alpha,bool wor,bool include_best,bool add_bos,bool add_eos,bool reverse,bool emit_unk_piece){ + RewriteIds(*self, static_cast(nullptr), + add_bos, add_eos, reverse, emit_unk_piece); + auto proto = self->SampleEncodeAndScoreAsImmutableProto(text, num_samples, + alpha, wor, include_best); + proto.ConvertToUnicodeSpans(); + return proto; + } +SWIGINTERN std::string sentencepiece_SentencePieceProcessor__Normalize(sentencepiece::SentencePieceProcessor *self,absl::string_view text){ + return self->Normalize(text); + } +SWIGINTERN std::pair< std::string,std::vector< size_t > > sentencepiece_SentencePieceProcessor__NormalizeWithOffsets(sentencepiece::SentencePieceProcessor *self,absl::string_view text){ + std::pair> result; + self->Normalize(text, &result.first, &result.second).IgnoreError(); + return result; + } +SWIGINTERN float sentencepiece_SentencePieceProcessor__CalculateEntropy(sentencepiece::SentencePieceProcessor *self,absl::string_view text,float alpha){ + return self->CalculateEntropy(text, alpha); + } +SWIGINTERN std::vector< float > sentencepiece_SentencePieceProcessor__CalculateEntropyBatch(sentencepiece::SentencePieceProcessor *self,std::vector< absl::string_view > const &ins,float alpha,int num_threads){ + std::vector outs(ins.size()); + InitNumThreads(ins, &num_threads); + { + ThreadPool pool(ins.size()); + std::atomic index = 0; + for (int n = 0; n < num_threads; ++n) { + pool.Schedule([&]() { + size_t i = 0; + while ((i = std::atomic_fetch_add(&index, 1)) < outs.size()) { + outs[i] = self->CalculateEntropy(ins[i], alpha); + } + }); + } + } + return outs; + } +SWIGINTERN sentencepiece::util::Status sentencepiece_SentencePieceProcessor__OverrideNormalizerSpec(sentencepiece::SentencePieceProcessor *self,std::unordered_map< std::string,std::string > const &args){ + sentencepiece::util::Status status; + for (const auto &[key, value] : args) { + status = sentencepiece::SentencePieceTrainer::SetProtoField( + key, value, + self->mutable_normalizer_spec()); + if (!status.ok()) return status; + } + return status; + } + +SWIGINTERN int +SWIG_AsVal_unsigned_SS_long (PyObject *obj, unsigned long *val) +{ +#if PY_VERSION_HEX < 0x03000000 + if (PyInt_Check(obj)) { + long v = PyInt_AsLong(obj); + if (v >= 0) { + if (val) *val = v; + return SWIG_OK; + } else { + return SWIG_OverflowError; + } + } else +#endif + if (PyLong_Check(obj)) { + unsigned long v = PyLong_AsUnsignedLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_OK; + } else { + PyErr_Clear(); + return SWIG_OverflowError; + } + } +#ifdef SWIG_PYTHON_CAST_MODE + { + int dispatch = 0; + unsigned long v = PyLong_AsUnsignedLong(obj); + if (!PyErr_Occurred()) { + if (val) *val = v; + return SWIG_AddCast(SWIG_OK); + } else { + PyErr_Clear(); + } + if (!dispatch) { + double d; + int res = SWIG_AddCast(SWIG_AsVal_double (obj,&d)); + // Largest double not larger than ULONG_MAX (not portably calculated easily) + // Note that double(ULONG_MAX) is stored in a double rounded up by one (for 64-bit unsigned long) + // 0xfffffffffffff800ULL == (uint64_t)std::nextafter(double(__uint128_t(ULONG_MAX)+1), double(0)) + const double ulong_max = sizeof(unsigned long) == 8 ? 0xfffffffffffff800ULL : ULONG_MAX; + if (SWIG_IsOK(res) && SWIG_CanCastAsInteger(&d, 0, ulong_max)) { + if (val) *val = (unsigned long)(d); + return res; + } + } + } +#endif + return SWIG_TypeError; +} + + +SWIGINTERN int +SWIG_AsVal_unsigned_SS_int (PyObject * obj, unsigned int *val) +{ + unsigned long v; + int res = SWIG_AsVal_unsigned_SS_long (obj, &v); + if (SWIG_IsOK(res)) { + if ((v > UINT_MAX)) { + return SWIG_OverflowError; + } else { + if (val) *val = static_cast< unsigned int >(v); + } + } + return res; +} + +SWIGINTERN void sentencepiece_SentencePieceTrainer__TrainFromString(absl::string_view arg){ + const auto _status = sentencepiece::SentencePieceTrainer::Train(arg); + if (!_status.ok()) throw _status; + return; + } +SWIGINTERN void sentencepiece_SentencePieceTrainer__TrainFromMap(std::unordered_map< std::string,std::string > const &args){ + const auto _status = sentencepiece::SentencePieceTrainer::Train(args); + if (!_status.ok()) throw _status; + return; + } +SWIGINTERN void sentencepiece_SentencePieceTrainer__TrainFromMap2(std::unordered_map< std::string,std::string > const &args,sentencepiece::SentenceIterator *iter){ + const auto _status = sentencepiece::SentencePieceTrainer::Train(args, iter); + if (!_status.ok()) throw _status; + return; + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceTrainer__TrainFromMap3(std::unordered_map< std::string,std::string > const &args){ + sentencepiece::util::bytes model_proto; + const auto _status = sentencepiece::SentencePieceTrainer::Train(args, nullptr, &model_proto); + if (!_status.ok()) throw _status; + return model_proto; + } +SWIGINTERN sentencepiece::util::bytes sentencepiece_SentencePieceTrainer__TrainFromMap4(std::unordered_map< std::string,std::string > const &args,sentencepiece::SentenceIterator *iter){ + sentencepiece::util::bytes model_proto; + const auto _status = sentencepiece::SentencePieceTrainer::Train(args, iter, &model_proto); + if (!_status.ok()) throw _status; + return model_proto; + } +SWIGINTERN sentencepiece::util::Status sentencepiece_SentencePieceNormalizer_LoadFromFile(sentencepiece::SentencePieceNormalizer *self,absl::string_view arg){ + return self->Load(arg); + } +SWIGINTERN std::string sentencepiece_SentencePieceNormalizer__Normalize(sentencepiece::SentencePieceNormalizer *self,absl::string_view text){ + std::string result; + const auto _status = self->Normalize(text, &result); + if (!_status.ok()) throw _status; + return result; + } +SWIGINTERN std::pair< std::string,std::vector< size_t > > sentencepiece_SentencePieceNormalizer__NormalizeWithOffsets(sentencepiece::SentencePieceNormalizer *self,absl::string_view text){ + std::pair> result; + const auto _status = self->Normalize(text, &result.first, &result.second); + if (!_status.ok()) throw _status; + return result; + } +SWIGINTERN void sentencepiece_SentencePieceNormalizer__SetProtoField(sentencepiece::SentencePieceNormalizer *self,absl::string_view name,bool value){ + sentencepiece::SentencePieceTrainer::SetProtoField( + name, + value ? "1" : "0", + self->mutable_normalizer_spec()).IgnoreError(); + } +#ifdef __cplusplus +extern "C" { +#endif +SWIGINTERN PyObject *_wrap_new_ImmutableSentencePieceText_ImmutableSentencePiece(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_ImmutableSentencePieceText_ImmutableSentencePiece", 0, 0, 0)) SWIG_fail; + { + try { + result = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *)new sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ImmutableSentencePieceText_ImmutableSentencePiece(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ImmutableSentencePieceText_ImmutableSentencePiece" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + delete arg1; + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__piece(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__piece" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + result = (std::string *) &((sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *)arg1)->piece(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(*result, input_type); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__surface(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__surface" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + result = (std::string *) &((sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *)arg1)->surface(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(*result, input_type); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__id(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + uint32_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__id" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + result = ((sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *)arg1)->id(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__begin(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + uint32_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__begin" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + result = ((sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *)arg1)->begin(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__end(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + uint32_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__end" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + result = ((sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *)arg1)->end(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_unsigned_SS_int(static_cast< unsigned int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::bytes *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + result = (sentencepiece::util::bytes *) &sentencepiece_ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes((sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *)arg1); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(*result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *arg1 = (sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::bytes *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece * >(argp1); + { + try { + result = (sentencepiece::util::bytes *) &sentencepiece_ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes((sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece const *)arg1); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(*result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ImmutableSentencePieceText_ImmutableSentencePiece_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ImmutableSentencePieceText_ImmutableSentencePiece_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_ImmutableSentencePieceText(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_ImmutableSentencePieceText", 0, 0, 0)) SWIG_fail; + { + try { + result = (sentencepiece::ImmutableSentencePieceText *)new sentencepiece::ImmutableSentencePieceText(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ImmutableSentencePieceText(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ImmutableSentencePieceText" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1); + { + try { + delete arg1; + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__pieces_size(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + size_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__pieces_size" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1); + { + try { + result = ((sentencepiece::ImmutableSentencePieceText const *)arg1)->pieces_size(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__pieces(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ImmutableSentencePieceText__pieces", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__pieces" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableSentencePieceText__pieces" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = ((sentencepiece::ImmutableSentencePieceText const *)arg1)->pieces(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece(result)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__text(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__text" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1); + { + try { + result = (std::string *) &((sentencepiece::ImmutableSentencePieceText const *)arg1)->text(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(*result, input_type); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__score(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + float result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__score" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1); + { + try { + result = (float)((sentencepiece::ImmutableSentencePieceText const *)arg1)->score(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_float(static_cast< float >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText_SerializeAsString(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::bytes result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText_SerializeAsString" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1); + { + try { + result = ((sentencepiece::ImmutableSentencePieceText const *)arg1)->SerializeAsString(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableSentencePieceText__text_as_bytes(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableSentencePieceText *arg1 = (sentencepiece::ImmutableSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::bytes *result = 0 ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableSentencePieceText__text_as_bytes" "', argument " "1"" of type '" "sentencepiece::ImmutableSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableSentencePieceText * >(argp1); + { + try { + result = (sentencepiece::util::bytes *) &sentencepiece_ImmutableSentencePieceText__text_as_bytes((sentencepiece::ImmutableSentencePieceText const *)arg1); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(*result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ImmutableSentencePieceText_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ImmutableSentencePieceText_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_ImmutableNBestSentencePieceText(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableNBestSentencePieceText *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_ImmutableNBestSentencePieceText", 0, 0, 0)) SWIG_fail; + { + try { + result = (sentencepiece::ImmutableNBestSentencePieceText *)new sentencepiece::ImmutableNBestSentencePieceText(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_ImmutableNBestSentencePieceText(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableNBestSentencePieceText *arg1 = (sentencepiece::ImmutableNBestSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_ImmutableNBestSentencePieceText" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableNBestSentencePieceText * >(argp1); + { + try { + delete arg1; + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText__nbests_size(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableNBestSentencePieceText *arg1 = (sentencepiece::ImmutableNBestSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + size_t result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText__nbests_size" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableNBestSentencePieceText * >(argp1); + { + try { + result = ((sentencepiece::ImmutableNBestSentencePieceText const *)arg1)->nbests_size(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_size_t(static_cast< size_t >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText__nbests(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableNBestSentencePieceText *arg1 = (sentencepiece::ImmutableNBestSentencePieceText *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::ImmutableSentencePieceText result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "ImmutableNBestSentencePieceText__nbests", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText__nbests" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableNBestSentencePieceText * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "ImmutableNBestSentencePieceText__nbests" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = ((sentencepiece::ImmutableNBestSentencePieceText const *)arg1)->nbests(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableSentencePieceText(result)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_ImmutableNBestSentencePieceText_SerializeAsString(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::ImmutableNBestSentencePieceText *arg1 = (sentencepiece::ImmutableNBestSentencePieceText *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::bytes result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "ImmutableNBestSentencePieceText_SerializeAsString" "', argument " "1"" of type '" "sentencepiece::ImmutableNBestSentencePieceText const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::ImmutableNBestSentencePieceText * >(argp1); + { + try { + result = ((sentencepiece::ImmutableNBestSentencePieceText const *)arg1)->SerializeAsString(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *ImmutableNBestSentencePieceText_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *ImmutableNBestSentencePieceText_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_new_SentencePieceProcessor(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SentencePieceProcessor", 0, 0, 0)) SWIG_fail; + { + try { + result = (sentencepiece::SentencePieceProcessor *)new sentencepiece::SentencePieceProcessor(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__SentencePieceProcessor, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SentencePieceProcessor(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SentencePieceProcessor" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + delete arg1; + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadFromSerializedProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_LoadFromSerializedProto", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_LoadFromSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = (arg1)->LoadFromSerializedProto(SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SetEncodeExtraOptions(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SetEncodeExtraOptions", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SetEncodeExtraOptions" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = (arg1)->SetEncodeExtraOptions(SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SetDecodeExtraOptions(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SetDecodeExtraOptions", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SetDecodeExtraOptions" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = (arg1)->SetDecodeExtraOptions(SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_SetVocabulary(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_SetVocabulary", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_SetVocabulary" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = (arg1)->SetVocabulary((std::vector< absl::string_view > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_ResetVocabulary(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::Status result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_ResetVocabulary" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + result = (arg1)->ResetVocabulary(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadVocabulary(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_LoadVocabulary", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_LoadVocabulary" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_LoadVocabulary" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = (arg1)->LoadVocabulary(SWIG_STD_MOVE(arg2),arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_CalculateEntropy__SWIG_0(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + float arg3 ; + float *arg4 = (float *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + float val3 ; + int ecode3 = 0 ; + void *argp4 = 0 ; + int res4 = 0 ; + sentencepiece::util::Status result; + + (void)self; + if ((nobjs < 4) || (nobjs > 4)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_float(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "3"" of type '" "float""'"); + } + arg3 = static_cast< float >(val3); + res4 = SWIG_ConvertPtr(swig_obj[3], &argp4,SWIGTYPE_p_float, 0 | 0 ); + if (!SWIG_IsOK(res4)) { + SWIG_exception_fail(SWIG_ArgError(res4), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "4"" of type '" "float *""'"); + } + arg4 = reinterpret_cast< float * >(argp4); + { + try { + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->CalculateEntropy(SWIG_STD_MOVE(arg2),arg3,arg4); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_CalculateEntropy__SWIG_1(PyObject *self, Py_ssize_t nobjs, PyObject **swig_obj) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + float arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + float val3 ; + int ecode3 = 0 ; + float result; + + (void)self; + if ((nobjs < 3) || (nobjs > 3)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_float(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor_CalculateEntropy" "', argument " "3"" of type '" "float""'"); + } + arg3 = static_cast< float >(val3); + { + try { + result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->CalculateEntropy(SWIG_STD_MOVE(arg2),arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_float(static_cast< float >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_CalculateEntropy(PyObject *self, PyObject *args) { + Py_ssize_t argc; + PyObject *argv[5] = { + 0 + }; + + if (!(argc = SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_CalculateEntropy", 0, 4, argv))) SWIG_fail; + --argc; + if (argc == 3) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_float(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + return _wrap_SentencePieceProcessor_CalculateEntropy__SWIG_1(self, argc, argv); + } + } + } + } + if (argc == 4) { + int _v = 0; + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[0], &vptr, SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0); + _v = SWIG_CheckState(res); + if (_v) { + int res = SWIG_AsCharPtrAndSize(argv[1], 0, NULL, 0); + _v = SWIG_CheckState(res); + if (_v) { + { + int res = SWIG_AsVal_float(argv[2], NULL); + _v = SWIG_CheckState(res); + } + if (_v) { + void *vptr = 0; + int res = SWIG_ConvertPtr(argv[3], &vptr, SWIGTYPE_p_float, 0); + _v = SWIG_CheckState(res); + if (_v) { + return _wrap_SentencePieceProcessor_CalculateEntropy__SWIG_0(self, argc, argv); + } + } + } + } + } + +fail: + SWIG_Python_RaiseOrModifyTypeError("Wrong number or type of arguments for overloaded function 'SentencePieceProcessor_CalculateEntropy'.\n" + " Possible C/C++ prototypes are:\n" + " sentencepiece::SentencePieceProcessor::CalculateEntropy(absl::string_view,float,float *) const\n" + " sentencepiece::SentencePieceProcessor::CalculateEntropy(absl::string_view,float) const\n"); + return 0; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetPieceSize(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + int result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_GetPieceSize" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->GetPieceSize(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_PieceToId(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + int result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_PieceToId", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_PieceToId" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->PieceToId(SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IdToPiece(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + std::string *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IdToPiece", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IdToPiece" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IdToPiece" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = (std::string *) &((sentencepiece::SentencePieceProcessor const *)arg1)->IdToPiece(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(*result, input_type); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_GetScore(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + float result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_GetScore", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_GetScore" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_GetScore" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = (float)((sentencepiece::SentencePieceProcessor const *)arg1)->GetScore(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_float(static_cast< float >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnknown(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + bool result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsUnknown", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsUnknown" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsUnknown" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnknown(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsControl(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + bool result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsControl", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsControl" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsControl" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsControl(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsUnused(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + bool result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsUnused", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsUnused" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsUnused" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsUnused(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_IsByte(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + int arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val2 ; + int ecode2 = 0 ; + PyObject *swig_obj[2] ; + bool result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_IsByte", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_IsByte" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + ecode2 = SWIG_AsVal_int(swig_obj[1], &val2); + if (!SWIG_IsOK(ecode2)) { + SWIG_exception_fail(SWIG_ArgError(ecode2), "in method '" "SentencePieceProcessor_IsByte" "', argument " "2"" of type '" "int""'"); + } + arg2 = static_cast< int >(val2); + { + try { + result = (bool)((sentencepiece::SentencePieceProcessor const *)arg1)->IsByte(arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_bool(static_cast< bool >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_unk_id(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + int result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_unk_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->unk_id(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_bos_id(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + int result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_bos_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->bos_id(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_eos_id(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + int result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_eos_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->eos_id(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_pad_id(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + int result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_pad_id" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + result = (int)((sentencepiece::SentencePieceProcessor const *)arg1)->pad_id(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_int(static_cast< int >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_serialized_model_proto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::bytes result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_serialized_model_proto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + try { + result = ((sentencepiece::SentencePieceProcessor const *)arg1)->serialized_model_proto(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor_LoadFromFile(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor_LoadFromFile", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor_LoadFromFile" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = sentencepiece_SentencePieceProcessor_LoadFromFile(arg1,SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsIds(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + bool arg3 ; + int arg4 ; + float arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + float val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + PyObject *swig_obj[9] ; + std::vector< int > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsIds", 9, 9, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "3"" of type '" "bool""'"); + } + arg3 = static_cast< bool >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "4"" of type '" "int""'"); + } + arg4 = static_cast< int >(val4); + ecode5 = SWIG_AsVal_float(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "5"" of type '" "float""'"); + } + arg5 = static_cast< float >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsIds" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsIds((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, PyInt_FromLong(static_cast(result[i]))); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsPieces(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + bool arg3 ; + int arg4 ; + float arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + float val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + PyObject *swig_obj[9] ; + std::vector< std::string > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsPieces", 9, 9, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "3"" of type '" "bool""'"); + } + arg3 = static_cast< bool >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "4"" of type '" "int""'"); + } + arg4 = static_cast< int >(val4); + ecode5 = SWIG_AsVal_float(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "5"" of type '" "float""'"); + } + arg5 = static_cast< float >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsPieces" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsPieces((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, MakePyOutputString(result[i], input_type)); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsSerializedProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + bool arg3 ; + int arg4 ; + float arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + float val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + PyObject *swig_obj[9] ; + sentencepiece::util::bytes result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsSerializedProto", 9, 9, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "3"" of type '" "bool""'"); + } + arg3 = static_cast< bool >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "4"" of type '" "int""'"); + } + arg4 = static_cast< int >(val4); + ecode5 = SWIG_AsVal_float(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "5"" of type '" "float""'"); + } + arg5 = static_cast< float >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsSerializedProto" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsImmutableProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + bool arg3 ; + int arg4 ; + float arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + float val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + PyObject *swig_obj[9] ; + sentencepiece::ImmutableSentencePieceText result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsImmutableProto", 9, 9, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "3"" of type '" "bool""'"); + } + arg3 = static_cast< bool >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "4"" of type '" "int""'"); + } + arg4 = static_cast< int >(val4); + ecode5 = SWIG_AsVal_float(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "5"" of type '" "float""'"); + } + arg5 = static_cast< float >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsImmutableProto" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsImmutableProto((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableSentencePieceText(result)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsIdsBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + int arg3 ; + bool arg4 ; + int arg5 ; + float arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + int val5 ; + int ecode5 = 0 ; + float val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + std::vector< std::vector< int > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsIdsBatch", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_int(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "5"" of type '" "int""'"); + } + arg5 = static_cast< int >(val5); + ecode6 = SWIG_AsVal_float(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "6"" of type '" "float""'"); + } + arg6 = static_cast< float >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__EncodeAsIdsBatch" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsIdsBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = PyList_New(result[i].size()); + for (size_t j = 0; j < result[i].size(); ++j) { + PyList_SET_ITEM(obj, j, PyInt_FromLong(static_cast(result[i][j]))); + } + PyList_SET_ITEM(resultobj, i, obj); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsPiecesBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + int arg3 ; + bool arg4 ; + int arg5 ; + float arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + int val5 ; + int ecode5 = 0 ; + float val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + std::vector< std::vector< std::string > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsPiecesBatch", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_int(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "5"" of type '" "int""'"); + } + arg5 = static_cast< int >(val5); + ecode6 = SWIG_AsVal_float(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "6"" of type '" "float""'"); + } + arg6 = static_cast< float >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__EncodeAsPiecesBatch" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsPiecesBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = PyList_New(result[i].size()); + for (size_t j = 0; j < result[i].size(); ++j) { + PyList_SET_ITEM(obj, j, MakePyOutputString(result[i][j], input_type)); + } + PyList_SET_ITEM(resultobj, i, obj); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsSerializedProtoBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + int arg3 ; + bool arg4 ; + int arg5 ; + float arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + int val5 ; + int ecode5 = 0 ; + float val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + BytesArray result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsSerializedProtoBatch", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_int(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "5"" of type '" "int""'"); + } + arg5 = static_cast< int >(val5); + ecode6 = SWIG_AsVal_float(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "6"" of type '" "float""'"); + } + arg6 = static_cast< float >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__EncodeAsSerializedProtoBatch" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsSerializedProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, MakePyOutputBytes(result[i])); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__EncodeAsImmutableProtoBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + int arg3 ; + bool arg4 ; + int arg5 ; + float arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + int val5 ; + int ecode5 = 0 ; + float val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + SwigValueWrapper< std::vector< sentencepiece::ImmutableSentencePieceText > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__EncodeAsImmutableProtoBatch", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_int(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "5"" of type '" "int""'"); + } + arg5 = static_cast< int >(val5); + ecode6 = SWIG_AsVal_float(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "6"" of type '" "float""'"); + } + arg6 = static_cast< float >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__EncodeAsImmutableProtoBatch" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__EncodeAsImmutableProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = SWIG_NewPointerObj(new sentencepiece::ImmutableSentencePieceText((&result)->at(i)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0); + PyList_SET_ITEM(resultobj, i, obj); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIds(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< int > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + std::string result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIds", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyInt_Check(o)) { + (*out)[i] = static_cast(PyInt_AsLong(o)); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIds((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(result, input_type); + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsBytes(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< int > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::bytes result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsBytes", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsBytes" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyInt_Check(o)) { + (*out)[i] = static_cast(PyInt_AsLong(o)); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIdsAsBytes((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePieces(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + std::string result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePieces", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__DecodePieces((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(result, input_type); + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsSerializedProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< int > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::bytes result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsSerializedProto", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyInt_Check(o)) { + (*out)[i] = static_cast(PyInt_AsLong(o)); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesAsSerializedProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::bytes result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesAsSerializedProto", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsImmutableProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< int > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::ImmutableSentencePieceText result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsImmutableProto", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsImmutableProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyInt_Check(o)) { + (*out)[i] = static_cast(PyInt_AsLong(o)); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIdsAsImmutableProto((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< int > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableSentencePieceText(result)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0 ); + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesAsImmutableProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::ImmutableSentencePieceText result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesAsImmutableProto", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesAsImmutableProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__DecodePiecesAsImmutableProto((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< absl::string_view > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableSentencePieceText(result)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0 ); + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< std::vector< int > > *arg2 = 0 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + std::vector< std::string > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsBatch", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector> *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + PyObject *o2 = PyList_GetItem(o, j); + if (PyInt_Check(o2)) { + (*out)[i][j] = static_cast(PyInt_AsLong(o2)); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodeIdsBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIdsBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< int > > const &)*arg2,arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, MakePyOutputString(result[i], input_type)); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsBytesBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< std::vector< int > > *arg2 = 0 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + BytesArray result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsBytesBatch", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsBytesBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector> *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + PyObject *o2 = PyList_GetItem(o, j); + if (PyInt_Check(o2)) { + (*out)[i][j] = static_cast(PyInt_AsLong(o2)); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodeIdsAsBytesBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIdsAsBytesBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< int > > const &)*arg2,arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, MakePyOutputBytes(result[i])); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< std::vector< int > > *arg2 = 0 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + BytesArray result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector> *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + PyObject *o2 = PyList_GetItem(o, j); + if (PyInt_Check(o2)) { + (*out)[i][j] = static_cast(PyInt_AsLong(o2)); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< int > > const &)*arg2,arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, MakePyOutputBytes(result[i])); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< std::vector< int > > *arg2 = 0 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + SwigValueWrapper< std::vector< sentencepiece::ImmutableSentencePieceText > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector> *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + PyObject *o2 = PyList_GetItem(o, j); + if (PyInt_Check(o2)) { + (*out)[i][j] = static_cast(PyInt_AsLong(o2)); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = sentencepiece_SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< int > > const &)*arg2,arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = SWIG_NewPointerObj(new sentencepiece::ImmutableSentencePieceText((&result)->at(i)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0); + PyList_SET_ITEM(resultobj, i, obj); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< std::vector< absl::string_view > > *arg2 = 0 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + std::vector< std::string > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesBatch", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector> *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + const PyInputString ustring(PyList_GetItem(o, j)); + if (ustring.IsAvalable()) { + (*out)[i][j] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodePiecesBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = sentencepiece_SentencePieceProcessor__DecodePiecesBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< absl::string_view > > const &)*arg2,arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, MakePyOutputString(result[i], input_type)); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< std::vector< absl::string_view > > *arg2 = 0 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + BytesArray result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector> *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + const PyInputString ustring(PyList_GetItem(o, j)); + if (ustring.IsAvalable()) { + (*out)[i][j] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = sentencepiece_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< absl::string_view > > const &)*arg2,arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, MakePyOutputBytes(result[i])); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< std::vector< absl::string_view > > *arg2 = 0 ; + int arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + SwigValueWrapper< std::vector< sentencepiece::ImmutableSentencePieceText > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector> *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector>(size); + for (size_t i = 0; i < size; ++i) { + PyObject *o = PyList_GetItem(swig_obj[1], i); + if (PyList_Check(o)) { + const size_t size2 = PyList_Size(o); + (*out)[i].resize(size2); + for (size_t j = 0; j < size2; ++j) { + const PyInputString ustring(PyList_GetItem(o, j)); + if (ustring.IsAvalable()) { + (*out)[i][j] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError,"list must contain integers"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + } + } else { + PyErr_SetString(PyExc_TypeError,"not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + { + try { + result = sentencepiece_SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch((sentencepiece::SentencePieceProcessor const *)arg1,(std::vector< std::vector< absl::string_view > > const &)*arg2,arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = SWIG_NewPointerObj(new sentencepiece::ImmutableSentencePieceText((&result)->at(i)), SWIGTYPE_p_sentencepiece__ImmutableSentencePieceText, SWIG_POINTER_OWN | 0); + PyList_SET_ITEM(resultobj, i, obj); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsIds(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + bool arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + PyObject *swig_obj[7] ; + std::vector< std::vector< int > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsIds", 7, 7, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsIds" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + { + try { + result = sentencepiece_SentencePieceProcessor__NBestEncodeAsIds((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = PyList_New(result[i].size()); + for (size_t j = 0; j < result[i].size(); ++j) { + PyList_SET_ITEM(obj, j, PyInt_FromLong(static_cast(result[i][j]))); + } + PyList_SET_ITEM(resultobj, i, obj); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsPieces(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + bool arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + PyObject *swig_obj[7] ; + std::vector< std::vector< std::string > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsPieces", 7, 7, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsPieces" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + { + try { + result = sentencepiece_SentencePieceProcessor__NBestEncodeAsPieces((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = PyList_New(result[i].size()); + for (size_t j = 0; j < result[i].size(); ++j) { + PyList_SET_ITEM(obj, j, MakePyOutputString(result[i][j], input_type)); + } + PyList_SET_ITEM(resultobj, i, obj); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsSerializedProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + bool arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + PyObject *swig_obj[7] ; + sentencepiece::util::bytes result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsSerializedProto", 7, 7, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsSerializedProto" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + { + try { + result = sentencepiece_SentencePieceProcessor__NBestEncodeAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NBestEncodeAsImmutableProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + bool arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + bool val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + PyObject *swig_obj[7] ; + sentencepiece::ImmutableNBestSentencePieceText result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NBestEncodeAsImmutableProto", 7, 7, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NBestEncodeAsImmutableProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__NBestEncodeAsImmutableProto" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_bool(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__NBestEncodeAsImmutableProto" "', argument " "4"" of type '" "bool""'"); + } + arg4 = static_cast< bool >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__NBestEncodeAsImmutableProto" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__NBestEncodeAsImmutableProto" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__NBestEncodeAsImmutableProto" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + { + try { + result = sentencepiece_SentencePieceProcessor__NBestEncodeAsImmutableProto((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableNBestSentencePieceText(result)), SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__SampleEncodeAndScoreAsIds(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + float arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + float val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + std::vector< std::pair< std::vector< int >,float > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__SampleEncodeAndScoreAsIds", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_float(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "4"" of type '" "float""'"); + } + arg4 = static_cast< float >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsIds" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsIds((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = PyList_New(result[i].first.size()); + for (size_t j = 0; j < result[i].first.size(); ++j) { + PyList_SET_ITEM(obj, j, PyInt_FromLong(static_cast(result[i].first[j]))); + } + PyList_SET_ITEM(resultobj, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast(result[i].second)))); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__SampleEncodeAndScoreAsPieces(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + float arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + float val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + std::vector< std::pair< std::vector< std::string >,float > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__SampleEncodeAndScoreAsPieces", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_float(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "4"" of type '" "float""'"); + } + arg4 = static_cast< float >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsPieces" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsPieces((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyObject *obj = PyList_New(result[i].first.size()); + for (size_t j = 0; j < result[i].first.size(); ++j) { + PyList_SET_ITEM(obj, j, MakePyOutputString(result[i].first[j], input_type)); + } + PyList_SET_ITEM(resultobj, i, PyTuple_Pack(2, obj, PyFloat_FromDouble(static_cast(result[i].second)))); + } + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + float arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + float val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + sentencepiece::util::bytes result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_float(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "4"" of type '" "float""'"); + } + arg4 = static_cast< float >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + int arg3 ; + float arg4 ; + bool arg5 ; + bool arg6 ; + bool arg7 ; + bool arg8 ; + bool arg9 ; + bool arg10 ; + void *argp1 = 0 ; + int res1 = 0 ; + int val3 ; + int ecode3 = 0 ; + float val4 ; + int ecode4 = 0 ; + bool val5 ; + int ecode5 = 0 ; + bool val6 ; + int ecode6 = 0 ; + bool val7 ; + int ecode7 = 0 ; + bool val8 ; + int ecode8 = 0 ; + bool val9 ; + int ecode9 = 0 ; + bool val10 ; + int ecode10 = 0 ; + PyObject *swig_obj[10] ; + sentencepiece::ImmutableNBestSentencePieceText result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto", 10, 10, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_int(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "3"" of type '" "int""'"); + } + arg3 = static_cast< int >(val3); + ecode4 = SWIG_AsVal_float(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "4"" of type '" "float""'"); + } + arg4 = static_cast< float >(val4); + ecode5 = SWIG_AsVal_bool(swig_obj[4], &val5); + if (!SWIG_IsOK(ecode5)) { + SWIG_exception_fail(SWIG_ArgError(ecode5), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "5"" of type '" "bool""'"); + } + arg5 = static_cast< bool >(val5); + ecode6 = SWIG_AsVal_bool(swig_obj[5], &val6); + if (!SWIG_IsOK(ecode6)) { + SWIG_exception_fail(SWIG_ArgError(ecode6), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "6"" of type '" "bool""'"); + } + arg6 = static_cast< bool >(val6); + ecode7 = SWIG_AsVal_bool(swig_obj[6], &val7); + if (!SWIG_IsOK(ecode7)) { + SWIG_exception_fail(SWIG_ArgError(ecode7), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "7"" of type '" "bool""'"); + } + arg7 = static_cast< bool >(val7); + ecode8 = SWIG_AsVal_bool(swig_obj[7], &val8); + if (!SWIG_IsOK(ecode8)) { + SWIG_exception_fail(SWIG_ArgError(ecode8), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "8"" of type '" "bool""'"); + } + arg8 = static_cast< bool >(val8); + ecode9 = SWIG_AsVal_bool(swig_obj[8], &val9); + if (!SWIG_IsOK(ecode9)) { + SWIG_exception_fail(SWIG_ArgError(ecode9), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "9"" of type '" "bool""'"); + } + arg9 = static_cast< bool >(val9); + ecode10 = SWIG_AsVal_bool(swig_obj[9], &val10); + if (!SWIG_IsOK(ecode10)) { + SWIG_exception_fail(SWIG_ArgError(ecode10), "in method '" "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto" "', argument " "10"" of type '" "bool""'"); + } + arg10 = static_cast< bool >(val10); + { + try { + result = sentencepiece_SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto((sentencepiece::SentencePieceProcessor const *)arg1,SWIG_STD_MOVE(arg2),arg3,arg4,arg5,arg6,arg7,arg8,arg9,arg10); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj((new sentencepiece::ImmutableNBestSentencePieceText(result)), SWIGTYPE_p_sentencepiece__ImmutableNBestSentencePieceText, SWIG_POINTER_OWN | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__Normalize(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + std::string result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__Normalize", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__Normalize" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = sentencepiece_SentencePieceProcessor__Normalize(arg1,SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(result, input_type); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__NormalizeWithOffsets(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + std::pair< std::string,std::vector< size_t > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__NormalizeWithOffsets", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__NormalizeWithOffsets" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = sentencepiece_SentencePieceProcessor__NormalizeWithOffsets(arg1,SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + if (PyInputString::IsUnicode(input_type)) { + sentencepiece::ConvertToUnicodeAlignment(arg2, (&result)->first, &(&result)->second); + } + PyObject *obj = PyList_New((&result)->second.size()); + for (size_t i = 0; i < (&result)->second.size(); ++i) { + PyList_SET_ITEM(obj, i, PyInt_FromLong(static_cast((&result)->second[i]))); + } + resultobj = PyTuple_Pack(2, MakePyOutputString((&result)->first, input_type), obj); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__CalculateEntropy(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + absl::string_view arg2 ; + float arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + float val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + float result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__CalculateEntropy", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__CalculateEntropy" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_float(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__CalculateEntropy" "', argument " "3"" of type '" "float""'"); + } + arg3 = static_cast< float >(val3); + { + try { + result = (float)sentencepiece_SentencePieceProcessor__CalculateEntropy(arg1,SWIG_STD_MOVE(arg2),arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_From_float(static_cast< float >(result)); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__CalculateEntropyBatch(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::vector< absl::string_view > *arg2 = 0 ; + float arg3 ; + int arg4 ; + void *argp1 = 0 ; + int res1 = 0 ; + float val3 ; + int ecode3 = 0 ; + int val4 ; + int ecode4 = 0 ; + PyObject *swig_obj[4] ; + std::vector< float > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__CalculateEntropyBatch", 4, 4, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__CalculateEntropyBatch" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::vector *out = nullptr; + if (PyList_Check(swig_obj[1])) { + const size_t size = PyList_Size(swig_obj[1]); + out = new std::vector(size); + for (size_t i = 0; i < size; ++i) { + const PyInputString ustring(PyList_GetItem(swig_obj[1], i)); + if (ustring.IsAvalable()) { + (*out)[i] = ustring.str(); + } else { + PyErr_SetString(PyExc_TypeError, "list must contain strings"); + SWIG_fail; + } + resultobj = ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a list"); + SWIG_fail; + } + arg2 = out; + } + ecode3 = SWIG_AsVal_float(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceProcessor__CalculateEntropyBatch" "', argument " "3"" of type '" "float""'"); + } + arg3 = static_cast< float >(val3); + ecode4 = SWIG_AsVal_int(swig_obj[3], &val4); + if (!SWIG_IsOK(ecode4)) { + SWIG_exception_fail(SWIG_ArgError(ecode4), "in method '" "SentencePieceProcessor__CalculateEntropyBatch" "', argument " "4"" of type '" "int""'"); + } + arg4 = static_cast< int >(val4); + { + try { + result = sentencepiece_SentencePieceProcessor__CalculateEntropyBatch(arg1,(std::vector< absl::string_view > const &)*arg2,arg3,arg4); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = PyList_New((&result)->size()); + for (size_t i = 0; i < (&result)->size(); ++i) { + PyList_SET_ITEM(resultobj, i, PyFloat_FromDouble(static_cast(result[i]))); + } + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceProcessor__OverrideNormalizerSpec(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceProcessor *arg1 = (sentencepiece::SentencePieceProcessor *) 0 ; + std::unordered_map< std::string,std::string > *arg2 = 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceProcessor__OverrideNormalizerSpec", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceProcessor, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceProcessor__OverrideNormalizerSpec" "', argument " "1"" of type '" "sentencepiece::SentencePieceProcessor *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceProcessor * >(argp1); + { + std::unordered_map *out = nullptr; + if (PyDict_Check(swig_obj[1])) { + PyObject *key, *value; + Py_ssize_t pos = 0; + out = new std::unordered_map; + while (PyDict_Next(swig_obj[1], &pos, &key, &value)) { + const PyInputString key_ustring(key); + const PyInputString value_ustring(value); + if (key_ustring.IsAvalable() && value_ustring.IsAvalable()) { + out->emplace(std::string(key_ustring.data(), key_ustring.size()), + std::string(value_ustring.data(), value_ustring.size())); + } else { + PyErr_SetString(PyExc_TypeError, "map must contain strings."); + SWIG_fail; + } + resultobj = key_ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a dictionary"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceProcessor__OverrideNormalizerSpec(arg1,(std::unordered_map< std::string,std::string > const &)*arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *SentencePieceProcessor_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_sentencepiece__SentencePieceProcessor, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SentencePieceProcessor_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_SetRandomGeneratorSeed(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + unsigned int arg1 ; + unsigned int val1 ; + int ecode1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + ecode1 = SWIG_AsVal_unsigned_SS_int(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "SetRandomGeneratorSeed" "', argument " "1"" of type '" "unsigned int""'"); + } + arg1 = static_cast< unsigned int >(val1); + { + try { + sentencepiece::SetRandomGeneratorSeed(arg1); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SetMinLogLevel(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + int arg1 ; + int val1 ; + int ecode1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + ecode1 = SWIG_AsVal_int(swig_obj[0], &val1); + if (!SWIG_IsOK(ecode1)) { + SWIG_exception_fail(SWIG_ArgError(ecode1), "in method '" "SetMinLogLevel" "', argument " "1"" of type '" "int""'"); + } + arg1 = static_cast< int >(val1); + { + try { + sentencepiece::SetMinLogLevel(arg1); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceTrainer__TrainFromString(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + absl::string_view arg1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + const PyInputString ustring(swig_obj[0]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg1 = ustring.str(); + } + { + try { + sentencepiece_SentencePieceTrainer__TrainFromString(SWIG_STD_MOVE(arg1)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceTrainer__TrainFromMap(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::unordered_map< std::string,std::string > *arg1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + std::unordered_map *out = nullptr; + if (PyDict_Check(swig_obj[0])) { + PyObject *key, *value; + Py_ssize_t pos = 0; + out = new std::unordered_map; + while (PyDict_Next(swig_obj[0], &pos, &key, &value)) { + const PyInputString key_ustring(key); + const PyInputString value_ustring(value); + if (key_ustring.IsAvalable() && value_ustring.IsAvalable()) { + out->emplace(std::string(key_ustring.data(), key_ustring.size()), + std::string(value_ustring.data(), value_ustring.size())); + } else { + PyErr_SetString(PyExc_TypeError, "map must contain strings."); + SWIG_fail; + } + resultobj = key_ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a dictionary"); + SWIG_fail; + } + arg1 = out; + } + { + try { + sentencepiece_SentencePieceTrainer__TrainFromMap((std::unordered_map< std::string,std::string > const &)*arg1); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + { + delete arg1; + } + return resultobj; +fail: + { + delete arg1; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceTrainer__TrainFromMap2(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::unordered_map< std::string,std::string > *arg1 = 0 ; + sentencepiece::SentenceIterator *arg2 = (sentencepiece::SentenceIterator *) 0 ; + PyObject *swig_obj[2] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceTrainer__TrainFromMap2", 2, 2, swig_obj)) SWIG_fail; + { + std::unordered_map *out = nullptr; + if (PyDict_Check(swig_obj[0])) { + PyObject *key, *value; + Py_ssize_t pos = 0; + out = new std::unordered_map; + while (PyDict_Next(swig_obj[0], &pos, &key, &value)) { + const PyInputString key_ustring(key); + const PyInputString value_ustring(value); + if (key_ustring.IsAvalable() && value_ustring.IsAvalable()) { + out->emplace(std::string(key_ustring.data(), key_ustring.size()), + std::string(value_ustring.data(), value_ustring.size())); + } else { + PyErr_SetString(PyExc_TypeError, "map must contain strings."); + SWIG_fail; + } + resultobj = key_ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a dictionary"); + SWIG_fail; + } + arg1 = out; + } + { + sentencepiece::SentenceIterator *out = nullptr; + if (PyIter_Check(swig_obj[1])) { + out = new PySentenceIterator(swig_obj[1]); + } else { + PyErr_SetString(PyExc_TypeError, "not a iterator"); + SWIG_fail; + } + arg2 = out; + } + { + try { + sentencepiece_SentencePieceTrainer__TrainFromMap2((std::unordered_map< std::string,std::string > const &)*arg1,arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + { + delete arg1; + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg1; + } + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceTrainer__TrainFromMap3(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::unordered_map< std::string,std::string > *arg1 = 0 ; + PyObject *swig_obj[1] ; + sentencepiece::util::bytes result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + std::unordered_map *out = nullptr; + if (PyDict_Check(swig_obj[0])) { + PyObject *key, *value; + Py_ssize_t pos = 0; + out = new std::unordered_map; + while (PyDict_Next(swig_obj[0], &pos, &key, &value)) { + const PyInputString key_ustring(key); + const PyInputString value_ustring(value); + if (key_ustring.IsAvalable() && value_ustring.IsAvalable()) { + out->emplace(std::string(key_ustring.data(), key_ustring.size()), + std::string(value_ustring.data(), value_ustring.size())); + } else { + PyErr_SetString(PyExc_TypeError, "map must contain strings."); + SWIG_fail; + } + resultobj = key_ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a dictionary"); + SWIG_fail; + } + arg1 = out; + } + { + try { + result = sentencepiece_SentencePieceTrainer__TrainFromMap3((std::unordered_map< std::string,std::string > const &)*arg1); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + { + delete arg1; + } + return resultobj; +fail: + { + delete arg1; + } + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceTrainer__TrainFromMap4(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + std::unordered_map< std::string,std::string > *arg1 = 0 ; + sentencepiece::SentenceIterator *arg2 = (sentencepiece::SentenceIterator *) 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::bytes result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceTrainer__TrainFromMap4", 2, 2, swig_obj)) SWIG_fail; + { + std::unordered_map *out = nullptr; + if (PyDict_Check(swig_obj[0])) { + PyObject *key, *value; + Py_ssize_t pos = 0; + out = new std::unordered_map; + while (PyDict_Next(swig_obj[0], &pos, &key, &value)) { + const PyInputString key_ustring(key); + const PyInputString value_ustring(value); + if (key_ustring.IsAvalable() && value_ustring.IsAvalable()) { + out->emplace(std::string(key_ustring.data(), key_ustring.size()), + std::string(value_ustring.data(), value_ustring.size())); + } else { + PyErr_SetString(PyExc_TypeError, "map must contain strings."); + SWIG_fail; + } + resultobj = key_ustring.input_type(); + } + } else { + PyErr_SetString(PyExc_TypeError, "not a dictionary"); + SWIG_fail; + } + arg1 = out; + } + { + sentencepiece::SentenceIterator *out = nullptr; + if (PyIter_Check(swig_obj[1])) { + out = new PySentenceIterator(swig_obj[1]); + } else { + PyErr_SetString(PyExc_TypeError, "not a iterator"); + SWIG_fail; + } + arg2 = out; + } + { + try { + result = sentencepiece_SentencePieceTrainer__TrainFromMap4((std::unordered_map< std::string,std::string > const &)*arg1,arg2); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + resultobj = MakePyOutputBytes(result); + } + { + delete arg1; + } + { + delete arg2; + } + return resultobj; +fail: + { + delete arg1; + } + { + delete arg2; + } + return NULL; +} + + +SWIGINTERN PyObject *SentencePieceTrainer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_sentencepiece__SentencePieceTrainer, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *_wrap_new_SentencePieceNormalizer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *result = 0 ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "new_SentencePieceNormalizer", 0, 0, 0)) SWIG_fail; + { + try { + result = (sentencepiece::SentencePieceNormalizer *)new sentencepiece::SentencePieceNormalizer(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_NewPointerObj(SWIG_as_voidptr(result), SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, SWIG_POINTER_NEW | 0 ); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_delete_SentencePieceNormalizer(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, SWIG_POINTER_DISOWN | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "delete_SentencePieceNormalizer" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + try { + delete arg1; + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer_LoadFromSerializedProto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceNormalizer_LoadFromSerializedProto", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer_LoadFromSerializedProto" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = (arg1)->LoadFromSerializedProto(SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer_LoadFromRuleTSV(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceNormalizer_LoadFromRuleTSV", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer_LoadFromRuleTSV" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = (arg1)->LoadFromRuleTSV(SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer_LoadFromRuleName(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceNormalizer_LoadFromRuleName", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer_LoadFromRuleName" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = (arg1)->LoadFromRuleName(SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer_serialized_model_proto(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[1] ; + std::string result; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer_serialized_model_proto" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer const *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + try { + result = ((sentencepiece::SentencePieceNormalizer const *)arg1)->serialized_model_proto(); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(result, input_type); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer_LoadFromFile(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + sentencepiece::util::Status result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceNormalizer_LoadFromFile", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer_LoadFromFile" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = sentencepiece_SentencePieceNormalizer_LoadFromFile(arg1,SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + if (!(&result)->ok()) { + SWIG_exception(ToSwigError((&result)->code()), (&result)->ToString().c_str()); + } + resultobj = SWIG_From_bool((&result)->ok()); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer__Normalize(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + std::string result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceNormalizer__Normalize", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer__Normalize" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = sentencepiece_SentencePieceNormalizer__Normalize(arg1,SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + resultobj = MakePyOutputString(result, input_type); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer__NormalizeWithOffsets(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + absl::string_view arg2 ; + void *argp1 = 0 ; + int res1 = 0 ; + PyObject *swig_obj[2] ; + std::pair< std::string,std::vector< size_t > > result; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceNormalizer__NormalizeWithOffsets", 2, 2, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer__NormalizeWithOffsets" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + { + try { + result = sentencepiece_SentencePieceNormalizer__NormalizeWithOffsets(arg1,SWIG_STD_MOVE(arg2)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + { + PyObject *input_type = resultobj; + if (PyInputString::IsUnicode(input_type)) { + sentencepiece::ConvertToUnicodeAlignment(arg2, (&result)->first, &(&result)->second); + } + PyObject *obj = PyList_New((&result)->second.size()); + for (size_t i = 0; i < (&result)->second.size(); ++i) { + PyList_SET_ITEM(obj, i, PyInt_FromLong(static_cast((&result)->second[i]))); + } + resultobj = PyTuple_Pack(2, MakePyOutputString((&result)->first, input_type), obj); + } + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *_wrap_SentencePieceNormalizer__SetProtoField(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + sentencepiece::SentencePieceNormalizer *arg1 = (sentencepiece::SentencePieceNormalizer *) 0 ; + absl::string_view arg2 ; + bool arg3 ; + void *argp1 = 0 ; + int res1 = 0 ; + bool val3 ; + int ecode3 = 0 ; + PyObject *swig_obj[3] ; + + (void)self; + if (!SWIG_Python_UnpackTuple(args, "SentencePieceNormalizer__SetProtoField", 3, 3, swig_obj)) SWIG_fail; + res1 = SWIG_ConvertPtr(swig_obj[0], &argp1,SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, 0 | 0 ); + if (!SWIG_IsOK(res1)) { + SWIG_exception_fail(SWIG_ArgError(res1), "in method '" "SentencePieceNormalizer__SetProtoField" "', argument " "1"" of type '" "sentencepiece::SentencePieceNormalizer *""'"); + } + arg1 = reinterpret_cast< sentencepiece::SentencePieceNormalizer * >(argp1); + { + const PyInputString ustring(swig_obj[1]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg2 = ustring.str(); + } + ecode3 = SWIG_AsVal_bool(swig_obj[2], &val3); + if (!SWIG_IsOK(ecode3)) { + SWIG_exception_fail(SWIG_ArgError(ecode3), "in method '" "SentencePieceNormalizer__SetProtoField" "', argument " "3"" of type '" "bool""'"); + } + arg3 = static_cast< bool >(val3); + { + try { + sentencepiece_SentencePieceNormalizer__SetProtoField(arg1,SWIG_STD_MOVE(arg2),arg3); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +SWIGINTERN PyObject *SentencePieceNormalizer_swigregister(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + PyObject *obj = NULL; + if (!SWIG_Python_UnpackTuple(args, "swigregister", 1, 1, &obj)) return NULL; + SWIG_TypeNewClientData(SWIGTYPE_p_sentencepiece__SentencePieceNormalizer, SWIG_NewClientData(obj)); + return SWIG_Py_Void(); +} + +SWIGINTERN PyObject *SentencePieceNormalizer_swiginit(PyObject *SWIGUNUSEDPARM(self), PyObject *args) { + return SWIG_Python_InitShadowInstance(args); +} + +SWIGINTERN PyObject *_wrap_SetDataDir(PyObject *self, PyObject *args) { + PyObject *resultobj = 0; + absl::string_view arg1 ; + PyObject *swig_obj[1] ; + + (void)self; + if (!args) SWIG_fail; + swig_obj[0] = args; + { + const PyInputString ustring(swig_obj[0]); + if (!ustring.IsAvalable()) { + PyErr_SetString(PyExc_TypeError, "not a string"); + SWIG_fail; + } + resultobj = ustring.input_type(); + arg1 = ustring.str(); + } + { + try { + sentencepiece::SetDataDir(SWIG_STD_MOVE(arg1)); + ReleaseResultObject(resultobj); + } + catch (const sentencepiece::util::Status &status) { + SWIG_exception(ToSwigError(status.code()), status.ToString().c_str()); + } + } + resultobj = SWIG_Py_Void(); + return resultobj; +fail: + return NULL; +} + + +static PyMethodDef SwigMethods[] = { + { "new_ImmutableSentencePieceText_ImmutableSentencePiece", _wrap_new_ImmutableSentencePieceText_ImmutableSentencePiece, METH_NOARGS, NULL}, + { "delete_ImmutableSentencePieceText_ImmutableSentencePiece", _wrap_delete_ImmutableSentencePieceText_ImmutableSentencePiece, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece__piece", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__piece, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece__surface", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__surface, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece__id", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__id, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece__begin", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__begin, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece__end", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__end, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__surface_as_bytes, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes", _wrap_ImmutableSentencePieceText_ImmutableSentencePiece__piece_as_bytes, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece_swigregister", ImmutableSentencePieceText_ImmutableSentencePiece_swigregister, METH_O, NULL}, + { "ImmutableSentencePieceText_ImmutableSentencePiece_swiginit", ImmutableSentencePieceText_ImmutableSentencePiece_swiginit, METH_VARARGS, NULL}, + { "new_ImmutableSentencePieceText", _wrap_new_ImmutableSentencePieceText, METH_NOARGS, NULL}, + { "delete_ImmutableSentencePieceText", _wrap_delete_ImmutableSentencePieceText, METH_O, NULL}, + { "ImmutableSentencePieceText__pieces_size", _wrap_ImmutableSentencePieceText__pieces_size, METH_O, NULL}, + { "ImmutableSentencePieceText__pieces", _wrap_ImmutableSentencePieceText__pieces, METH_VARARGS, NULL}, + { "ImmutableSentencePieceText__text", _wrap_ImmutableSentencePieceText__text, METH_O, NULL}, + { "ImmutableSentencePieceText__score", _wrap_ImmutableSentencePieceText__score, METH_O, NULL}, + { "ImmutableSentencePieceText_SerializeAsString", _wrap_ImmutableSentencePieceText_SerializeAsString, METH_O, NULL}, + { "ImmutableSentencePieceText__text_as_bytes", _wrap_ImmutableSentencePieceText__text_as_bytes, METH_O, NULL}, + { "ImmutableSentencePieceText_swigregister", ImmutableSentencePieceText_swigregister, METH_O, NULL}, + { "ImmutableSentencePieceText_swiginit", ImmutableSentencePieceText_swiginit, METH_VARARGS, NULL}, + { "new_ImmutableNBestSentencePieceText", _wrap_new_ImmutableNBestSentencePieceText, METH_NOARGS, NULL}, + { "delete_ImmutableNBestSentencePieceText", _wrap_delete_ImmutableNBestSentencePieceText, METH_O, NULL}, + { "ImmutableNBestSentencePieceText__nbests_size", _wrap_ImmutableNBestSentencePieceText__nbests_size, METH_O, NULL}, + { "ImmutableNBestSentencePieceText__nbests", _wrap_ImmutableNBestSentencePieceText__nbests, METH_VARARGS, NULL}, + { "ImmutableNBestSentencePieceText_SerializeAsString", _wrap_ImmutableNBestSentencePieceText_SerializeAsString, METH_O, NULL}, + { "ImmutableNBestSentencePieceText_swigregister", ImmutableNBestSentencePieceText_swigregister, METH_O, NULL}, + { "ImmutableNBestSentencePieceText_swiginit", ImmutableNBestSentencePieceText_swiginit, METH_VARARGS, NULL}, + { "new_SentencePieceProcessor", _wrap_new_SentencePieceProcessor, METH_NOARGS, NULL}, + { "delete_SentencePieceProcessor", _wrap_delete_SentencePieceProcessor, METH_O, NULL}, + { "SentencePieceProcessor_LoadFromSerializedProto", _wrap_SentencePieceProcessor_LoadFromSerializedProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor_SetEncodeExtraOptions", _wrap_SentencePieceProcessor_SetEncodeExtraOptions, METH_VARARGS, NULL}, + { "SentencePieceProcessor_SetDecodeExtraOptions", _wrap_SentencePieceProcessor_SetDecodeExtraOptions, METH_VARARGS, NULL}, + { "SentencePieceProcessor_SetVocabulary", _wrap_SentencePieceProcessor_SetVocabulary, METH_VARARGS, NULL}, + { "SentencePieceProcessor_ResetVocabulary", _wrap_SentencePieceProcessor_ResetVocabulary, METH_O, NULL}, + { "SentencePieceProcessor_LoadVocabulary", _wrap_SentencePieceProcessor_LoadVocabulary, METH_VARARGS, NULL}, + { "SentencePieceProcessor_CalculateEntropy", _wrap_SentencePieceProcessor_CalculateEntropy, METH_VARARGS, NULL}, + { "SentencePieceProcessor_GetPieceSize", _wrap_SentencePieceProcessor_GetPieceSize, METH_O, NULL}, + { "SentencePieceProcessor_PieceToId", _wrap_SentencePieceProcessor_PieceToId, METH_VARARGS, NULL}, + { "SentencePieceProcessor_IdToPiece", _wrap_SentencePieceProcessor_IdToPiece, METH_VARARGS, NULL}, + { "SentencePieceProcessor_GetScore", _wrap_SentencePieceProcessor_GetScore, METH_VARARGS, NULL}, + { "SentencePieceProcessor_IsUnknown", _wrap_SentencePieceProcessor_IsUnknown, METH_VARARGS, NULL}, + { "SentencePieceProcessor_IsControl", _wrap_SentencePieceProcessor_IsControl, METH_VARARGS, NULL}, + { "SentencePieceProcessor_IsUnused", _wrap_SentencePieceProcessor_IsUnused, METH_VARARGS, NULL}, + { "SentencePieceProcessor_IsByte", _wrap_SentencePieceProcessor_IsByte, METH_VARARGS, NULL}, + { "SentencePieceProcessor_unk_id", _wrap_SentencePieceProcessor_unk_id, METH_O, NULL}, + { "SentencePieceProcessor_bos_id", _wrap_SentencePieceProcessor_bos_id, METH_O, NULL}, + { "SentencePieceProcessor_eos_id", _wrap_SentencePieceProcessor_eos_id, METH_O, NULL}, + { "SentencePieceProcessor_pad_id", _wrap_SentencePieceProcessor_pad_id, METH_O, NULL}, + { "SentencePieceProcessor_serialized_model_proto", _wrap_SentencePieceProcessor_serialized_model_proto, METH_O, NULL}, + { "SentencePieceProcessor_LoadFromFile", _wrap_SentencePieceProcessor_LoadFromFile, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsIds", _wrap_SentencePieceProcessor__EncodeAsIds, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsPieces", _wrap_SentencePieceProcessor__EncodeAsPieces, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsSerializedProto", _wrap_SentencePieceProcessor__EncodeAsSerializedProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsImmutableProto", _wrap_SentencePieceProcessor__EncodeAsImmutableProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsIdsBatch", _wrap_SentencePieceProcessor__EncodeAsIdsBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsPiecesBatch", _wrap_SentencePieceProcessor__EncodeAsPiecesBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsSerializedProtoBatch", _wrap_SentencePieceProcessor__EncodeAsSerializedProtoBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__EncodeAsImmutableProtoBatch", _wrap_SentencePieceProcessor__EncodeAsImmutableProtoBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIds", _wrap_SentencePieceProcessor__DecodeIds, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIdsAsBytes", _wrap_SentencePieceProcessor__DecodeIdsAsBytes, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodePieces", _wrap_SentencePieceProcessor__DecodePieces, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIdsAsSerializedProto", _wrap_SentencePieceProcessor__DecodeIdsAsSerializedProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodePiecesAsSerializedProto", _wrap_SentencePieceProcessor__DecodePiecesAsSerializedProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIdsAsImmutableProto", _wrap_SentencePieceProcessor__DecodeIdsAsImmutableProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodePiecesAsImmutableProto", _wrap_SentencePieceProcessor__DecodePiecesAsImmutableProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIdsBatch", _wrap_SentencePieceProcessor__DecodeIdsBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIdsAsBytesBatch", _wrap_SentencePieceProcessor__DecodeIdsAsBytesBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch", _wrap_SentencePieceProcessor__DecodeIdsAsSerializedProtoBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch", _wrap_SentencePieceProcessor__DecodeIdsAsImmutableProtoBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodePiecesBatch", _wrap_SentencePieceProcessor__DecodePiecesBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch", _wrap_SentencePieceProcessor__DecodePiecesAsSerializedProtoBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch", _wrap_SentencePieceProcessor__DecodePiecesAsImmutableProtoBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__NBestEncodeAsIds", _wrap_SentencePieceProcessor__NBestEncodeAsIds, METH_VARARGS, NULL}, + { "SentencePieceProcessor__NBestEncodeAsPieces", _wrap_SentencePieceProcessor__NBestEncodeAsPieces, METH_VARARGS, NULL}, + { "SentencePieceProcessor__NBestEncodeAsSerializedProto", _wrap_SentencePieceProcessor__NBestEncodeAsSerializedProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__NBestEncodeAsImmutableProto", _wrap_SentencePieceProcessor__NBestEncodeAsImmutableProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__SampleEncodeAndScoreAsIds", _wrap_SentencePieceProcessor__SampleEncodeAndScoreAsIds, METH_VARARGS, NULL}, + { "SentencePieceProcessor__SampleEncodeAndScoreAsPieces", _wrap_SentencePieceProcessor__SampleEncodeAndScoreAsPieces, METH_VARARGS, NULL}, + { "SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto", _wrap_SentencePieceProcessor__SampleEncodeAndScoreAsSerializedProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto", _wrap_SentencePieceProcessor__SampleEncodeAndScoreAsImmutableProto, METH_VARARGS, NULL}, + { "SentencePieceProcessor__Normalize", _wrap_SentencePieceProcessor__Normalize, METH_VARARGS, NULL}, + { "SentencePieceProcessor__NormalizeWithOffsets", _wrap_SentencePieceProcessor__NormalizeWithOffsets, METH_VARARGS, NULL}, + { "SentencePieceProcessor__CalculateEntropy", _wrap_SentencePieceProcessor__CalculateEntropy, METH_VARARGS, NULL}, + { "SentencePieceProcessor__CalculateEntropyBatch", _wrap_SentencePieceProcessor__CalculateEntropyBatch, METH_VARARGS, NULL}, + { "SentencePieceProcessor__OverrideNormalizerSpec", _wrap_SentencePieceProcessor__OverrideNormalizerSpec, METH_VARARGS, NULL}, + { "SentencePieceProcessor_swigregister", SentencePieceProcessor_swigregister, METH_O, NULL}, + { "SentencePieceProcessor_swiginit", SentencePieceProcessor_swiginit, METH_VARARGS, NULL}, + { "SetRandomGeneratorSeed", _wrap_SetRandomGeneratorSeed, METH_O, NULL}, + { "SetMinLogLevel", _wrap_SetMinLogLevel, METH_O, NULL}, + { "SentencePieceTrainer__TrainFromString", _wrap_SentencePieceTrainer__TrainFromString, METH_O, NULL}, + { "SentencePieceTrainer__TrainFromMap", _wrap_SentencePieceTrainer__TrainFromMap, METH_O, NULL}, + { "SentencePieceTrainer__TrainFromMap2", _wrap_SentencePieceTrainer__TrainFromMap2, METH_VARARGS, NULL}, + { "SentencePieceTrainer__TrainFromMap3", _wrap_SentencePieceTrainer__TrainFromMap3, METH_O, NULL}, + { "SentencePieceTrainer__TrainFromMap4", _wrap_SentencePieceTrainer__TrainFromMap4, METH_VARARGS, NULL}, + { "SentencePieceTrainer_swigregister", SentencePieceTrainer_swigregister, METH_O, NULL}, + { "new_SentencePieceNormalizer", _wrap_new_SentencePieceNormalizer, METH_NOARGS, NULL}, + { "delete_SentencePieceNormalizer", _wrap_delete_SentencePieceNormalizer, METH_O, NULL}, + { "SentencePieceNormalizer_LoadFromSerializedProto", _wrap_SentencePieceNormalizer_LoadFromSerializedProto, METH_VARARGS, NULL}, + { "SentencePieceNormalizer_LoadFromRuleTSV", _wrap_SentencePieceNormalizer_LoadFromRuleTSV, METH_VARARGS, NULL}, + { "SentencePieceNormalizer_LoadFromRuleName", _wrap_SentencePieceNormalizer_LoadFromRuleName, METH_VARARGS, NULL}, + { "SentencePieceNormalizer_serialized_model_proto", _wrap_SentencePieceNormalizer_serialized_model_proto, METH_O, NULL}, + { "SentencePieceNormalizer_LoadFromFile", _wrap_SentencePieceNormalizer_LoadFromFile, METH_VARARGS, NULL}, + { "SentencePieceNormalizer__Normalize", _wrap_SentencePieceNormalizer__Normalize, METH_VARARGS, NULL}, + { "SentencePieceNormalizer__NormalizeWithOffsets", _wrap_SentencePieceNormalizer__NormalizeWithOffsets, METH_VARARGS, NULL}, + { "SentencePieceNormalizer__SetProtoField", _wrap_SentencePieceNormalizer__SetProtoField, METH_VARARGS, NULL}, + { "SentencePieceNormalizer_swigregister", SentencePieceNormalizer_swigregister, METH_O, NULL}, + { "SentencePieceNormalizer_swiginit", SentencePieceNormalizer_swiginit, METH_VARARGS, NULL}, + { "SetDataDir", _wrap_SetDataDir, METH_O, NULL}, + { NULL, NULL, 0, NULL } +}; + + +/* -------- TYPE CONVERSION AND EQUIVALENCE RULES (BEGIN) -------- */ + +static swig_type_info _swigt__p_char = {"_p_char", "char *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_float = {"_p_float", "float *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_sentencepiece__ImmutableNBestSentencePieceText = {"_p_sentencepiece__ImmutableNBestSentencePieceText", "sentencepiece::ImmutableNBestSentencePieceText *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_sentencepiece__ImmutableSentencePieceText = {"_p_sentencepiece__ImmutableSentencePieceText", "sentencepiece::ImmutableSentencePieceText *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece = {"_p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece", "sentencepiece::ImmutableSentencePieceText_ImmutableSentencePiece *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_sentencepiece__SentenceIterator = {"_p_sentencepiece__SentenceIterator", "sentencepiece::SentenceIterator *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_sentencepiece__SentencePieceNormalizer = {"_p_sentencepiece__SentencePieceNormalizer", "sentencepiece::SentencePieceNormalizer *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_sentencepiece__SentencePieceProcessor = {"_p_sentencepiece__SentencePieceProcessor", "sentencepiece::SentencePieceProcessor *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_sentencepiece__SentencePieceTrainer = {"_p_sentencepiece__SentencePieceTrainer", "sentencepiece::SentencePieceTrainer *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__string = {"_p_std__string", "sentencepiece::util::bytes *|std::string *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__unordered_mapT_std__string_std__string_t = {"_p_std__unordered_mapT_std__string_std__string_t", "std::unordered_map< std::string,std::string > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_absl__string_view_t = {"_p_std__vectorT_absl__string_view_t", "std::vector< absl::string_view > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_int_t = {"_p_std__vectorT_int_t", "std::vector< int > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_std__vectorT_absl__string_view_t_t = {"_p_std__vectorT_std__vectorT_absl__string_view_t_t", "std::vector< std::vector< absl::string_view > > *", 0, 0, (void*)0, 0}; +static swig_type_info _swigt__p_std__vectorT_std__vectorT_int_t_t = {"_p_std__vectorT_std__vectorT_int_t_t", "std::vector< std::vector< int > > *", 0, 0, (void*)0, 0}; + +static swig_type_info *swig_type_initial[] = { + &_swigt__p_char, + &_swigt__p_float, + &_swigt__p_sentencepiece__ImmutableNBestSentencePieceText, + &_swigt__p_sentencepiece__ImmutableSentencePieceText, + &_swigt__p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, + &_swigt__p_sentencepiece__SentenceIterator, + &_swigt__p_sentencepiece__SentencePieceNormalizer, + &_swigt__p_sentencepiece__SentencePieceProcessor, + &_swigt__p_sentencepiece__SentencePieceTrainer, + &_swigt__p_std__string, + &_swigt__p_std__unordered_mapT_std__string_std__string_t, + &_swigt__p_std__vectorT_absl__string_view_t, + &_swigt__p_std__vectorT_int_t, + &_swigt__p_std__vectorT_std__vectorT_absl__string_view_t_t, + &_swigt__p_std__vectorT_std__vectorT_int_t_t, +}; + +static swig_cast_info _swigc__p_char[] = { {&_swigt__p_char, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_float[] = { {&_swigt__p_float, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_sentencepiece__ImmutableNBestSentencePieceText[] = { {&_swigt__p_sentencepiece__ImmutableNBestSentencePieceText, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_sentencepiece__ImmutableSentencePieceText[] = { {&_swigt__p_sentencepiece__ImmutableSentencePieceText, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece[] = { {&_swigt__p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_sentencepiece__SentenceIterator[] = { {&_swigt__p_sentencepiece__SentenceIterator, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_sentencepiece__SentencePieceNormalizer[] = { {&_swigt__p_sentencepiece__SentencePieceNormalizer, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_sentencepiece__SentencePieceProcessor[] = { {&_swigt__p_sentencepiece__SentencePieceProcessor, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_sentencepiece__SentencePieceTrainer[] = { {&_swigt__p_sentencepiece__SentencePieceTrainer, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__string[] = { {&_swigt__p_std__string, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__unordered_mapT_std__string_std__string_t[] = { {&_swigt__p_std__unordered_mapT_std__string_std__string_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_absl__string_view_t[] = { {&_swigt__p_std__vectorT_absl__string_view_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_int_t[] = { {&_swigt__p_std__vectorT_int_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_std__vectorT_absl__string_view_t_t[] = { {&_swigt__p_std__vectorT_std__vectorT_absl__string_view_t_t, 0, 0, 0},{0, 0, 0, 0}}; +static swig_cast_info _swigc__p_std__vectorT_std__vectorT_int_t_t[] = { {&_swigt__p_std__vectorT_std__vectorT_int_t_t, 0, 0, 0},{0, 0, 0, 0}}; + +static swig_cast_info *swig_cast_initial[] = { + _swigc__p_char, + _swigc__p_float, + _swigc__p_sentencepiece__ImmutableNBestSentencePieceText, + _swigc__p_sentencepiece__ImmutableSentencePieceText, + _swigc__p_sentencepiece__ImmutableSentencePieceText_ImmutableSentencePiece, + _swigc__p_sentencepiece__SentenceIterator, + _swigc__p_sentencepiece__SentencePieceNormalizer, + _swigc__p_sentencepiece__SentencePieceProcessor, + _swigc__p_sentencepiece__SentencePieceTrainer, + _swigc__p_std__string, + _swigc__p_std__unordered_mapT_std__string_std__string_t, + _swigc__p_std__vectorT_absl__string_view_t, + _swigc__p_std__vectorT_int_t, + _swigc__p_std__vectorT_std__vectorT_absl__string_view_t_t, + _swigc__p_std__vectorT_std__vectorT_int_t_t, +}; + + +/* -------- TYPE CONVERSION AND EQUIVALENCE RULES (END) -------- */ + +static swig_const_info swig_const_table[] = { +{0, 0, 0, 0.0, 0, 0}}; + +#ifdef __cplusplus +} +#endif +/* ----------------------------------------------------------------------------- + * Type initialization: + * This problem is tough by the requirement that no dynamic + * memory is used. Also, since swig_type_info structures store pointers to + * swig_cast_info structures and swig_cast_info structures store pointers back + * to swig_type_info structures, we need some lookup code at initialization. + * The idea is that swig generates all the structures that are needed. + * The runtime then collects these partially filled structures. + * The SWIG_InitializeModule function takes these initial arrays out of + * swig_module, and does all the lookup, filling in the swig_module.types + * array with the correct data and linking the correct swig_cast_info + * structures together. + * + * The generated swig_type_info structures are assigned statically to an initial + * array. We just loop through that array, and handle each type individually. + * First we lookup if this type has been already loaded, and if so, use the + * loaded structure instead of the generated one. Then we have to fill in the + * cast linked list. The cast data is initially stored in something like a + * two-dimensional array. Each row corresponds to a type (there are the same + * number of rows as there are in the swig_type_initial array). Each entry in + * a column is one of the swig_cast_info structures for that type. + * The cast_initial array is actually an array of arrays, because each row has + * a variable number of columns. So to actually build the cast linked list, + * we find the array of casts associated with the type, and loop through it + * adding the casts to the list. The one last trick we need to do is making + * sure the type pointer in the swig_cast_info struct is correct. + * + * First off, we lookup the cast->type name to see if it is already loaded. + * There are three cases to handle: + * 1) If the cast->type has already been loaded AND the type we are adding + * casting info to has not been loaded (it is in this module), THEN we + * replace the cast->type pointer with the type pointer that has already + * been loaded. + * 2) If BOTH types (the one we are adding casting info to, and the + * cast->type) are loaded, THEN the cast info has already been loaded by + * the previous module so we just ignore it. + * 3) Finally, if cast->type has not already been loaded, then we add that + * swig_cast_info to the linked list (because the cast->type) pointer will + * be correct. + * ----------------------------------------------------------------------------- */ + +#ifdef __cplusplus +extern "C" { +#if 0 +} /* c-mode */ +#endif +#endif + +#if 0 +#define SWIGRUNTIME_DEBUG +#endif + +#ifndef SWIG_INIT_CLIENT_DATA_TYPE +#define SWIG_INIT_CLIENT_DATA_TYPE void * +#endif + +SWIGRUNTIME void +SWIG_InitializeModule(SWIG_INIT_CLIENT_DATA_TYPE clientdata) { + size_t i; + swig_module_info *module_head, *iter; + int init; + + /* check to see if the circular list has been setup, if not, set it up */ + if (swig_module.next==0) { + /* Initialize the swig_module */ + swig_module.type_initial = swig_type_initial; + swig_module.cast_initial = swig_cast_initial; + swig_module.next = &swig_module; + init = 1; + } else { + init = 0; + } + + /* Try and load any already created modules */ + module_head = SWIG_GetModule(clientdata); + if (!module_head) { + /* This is the first module loaded for this interpreter */ + /* so set the swig module into the interpreter */ + SWIG_SetModule(clientdata, &swig_module); + } else { + /* the interpreter has loaded a SWIG module, but has it loaded this one? */ + iter=module_head; + do { + if (iter==&swig_module) { + /* Our module is already in the list, so there's nothing more to do. */ + return; + } + iter=iter->next; + } while (iter!= module_head); + + /* otherwise we must add our module into the list */ + swig_module.next = module_head->next; + module_head->next = &swig_module; + } + + /* When multiple interpreters are used, a module could have already been initialized in + a different interpreter, but not yet have a pointer in this interpreter. + In this case, we do not want to continue adding types... everything should be + set up already */ + if (init == 0) return; + + /* Now work on filling in swig_module.types */ +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: size %lu\n", (unsigned long)swig_module.size); +#endif + for (i = 0; i < swig_module.size; ++i) { + swig_type_info *type = 0; + swig_type_info *ret; + swig_cast_info *cast; + +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: type %lu %s\n", (unsigned long)i, swig_module.type_initial[i]->name); +#endif + + /* if there is another module already loaded */ + if (swig_module.next != &swig_module) { + type = SWIG_MangledTypeQueryModule(swig_module.next, &swig_module, swig_module.type_initial[i]->name); + } + if (type) { + /* Overwrite clientdata field */ +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: found type %s\n", type->name); +#endif + if (swig_module.type_initial[i]->clientdata) { + type->clientdata = swig_module.type_initial[i]->clientdata; +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: found and overwrite type %s \n", type->name); +#endif + } + } else { + type = swig_module.type_initial[i]; + } + + /* Insert casting types */ + cast = swig_module.cast_initial[i]; + while (cast->type) { + /* Don't need to add information already in the list */ + ret = 0; +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: look cast %s\n", cast->type->name); +#endif + if (swig_module.next != &swig_module) { + ret = SWIG_MangledTypeQueryModule(swig_module.next, &swig_module, cast->type->name); +#ifdef SWIGRUNTIME_DEBUG + if (ret) printf("SWIG_InitializeModule: found cast %s\n", ret->name); +#endif + } + if (ret) { + if (type == swig_module.type_initial[i]) { +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: skip old type %s\n", ret->name); +#endif + cast->type = ret; + ret = 0; + } else { + /* Check for casting already in the list */ + swig_cast_info *ocast = SWIG_TypeCheck(ret->name, type); +#ifdef SWIGRUNTIME_DEBUG + if (ocast) printf("SWIG_InitializeModule: skip old cast %s\n", ret->name); +#endif + if (!ocast) ret = 0; + } + } + + if (!ret) { +#ifdef SWIGRUNTIME_DEBUG + printf("SWIG_InitializeModule: adding cast %s\n", cast->type->name); +#endif + if (type->cast) { + type->cast->prev = cast; + cast->next = type->cast; + } + type->cast = cast; + } + cast++; + } + /* Set entry in modules->types array equal to the type */ + swig_module.types[i] = type; + } + swig_module.types[i] = 0; + +#ifdef SWIGRUNTIME_DEBUG + printf("**** SWIG_InitializeModule: Cast List ******\n"); + for (i = 0; i < swig_module.size; ++i) { + int j = 0; + swig_cast_info *cast = swig_module.cast_initial[i]; + printf("SWIG_InitializeModule: type %lu %s\n", (unsigned long)i, swig_module.type_initial[i]->name); + while (cast->type) { + printf("SWIG_InitializeModule: cast type %s\n", cast->type->name); + cast++; + ++j; + } + printf("---- Total casts: %d\n",j); + } + printf("**** SWIG_InitializeModule: Cast List ******\n"); +#endif +} + +/* This function will propagate the clientdata field of type to +* any new swig_type_info structures that have been added into the list +* of equivalent types. It is like calling +* SWIG_TypeClientData(type, clientdata) a second time. +*/ +SWIGRUNTIME void +SWIG_PropagateClientData(void) { + size_t i; + swig_cast_info *equiv; + static int init_run = 0; + + if (init_run) return; + init_run = 1; + + for (i = 0; i < swig_module.size; i++) { + if (swig_module.types[i]->clientdata) { + equiv = swig_module.types[i]->cast; + while (equiv) { + if (!equiv->converter) { + if (equiv->type && !equiv->type->clientdata) + SWIG_TypeClientData(equiv->type, swig_module.types[i]->clientdata); + } + equiv = equiv->next; + } + } + } +} + +#ifdef __cplusplus +#if 0 +{ + /* c-mode */ +#endif +} +#endif + + + +#ifdef __cplusplus +extern "C" { +#endif + + /* ----------------------------------------------------------------------------- + * constants/methods manipulation + * ----------------------------------------------------------------------------- */ + + /* Install Constants */ + SWIGINTERN void + SWIG_Python_InstallConstants(PyObject *d, swig_const_info constants[]) { + PyObject *obj = 0; + size_t i; + for (i = 0; constants[i].type; ++i) { + switch(constants[i].type) { + case SWIG_PY_POINTER: + obj = SWIG_InternalNewPointerObj(constants[i].pvalue, *(constants[i]).ptype,0); + break; + case SWIG_PY_BINARY: + obj = SWIG_NewPackedObj(constants[i].pvalue, constants[i].lvalue, *(constants[i].ptype)); + break; + default: + obj = 0; + break; + } + if (obj) { + PyDict_SetItemString(d, constants[i].name, obj); + SWIG_Py_DECREF(obj); + } + } + } + + /* ----------------------------------------------------------------------------- + * Patch %callback methods' docstrings to hold the callback ptrs + * -----------------------------------------------------------------------------*/ + + SWIGINTERN void + SWIG_Python_FixMethods(PyMethodDef *methods, const swig_const_info *const_table, swig_type_info **types, swig_type_info **types_initial) { + size_t i; + for (i = 0; methods[i].ml_name; ++i) { + const char *c = methods[i].ml_doc; + if (!c) continue; + c = strstr(c, "swig_ptr: "); + if (c) { + int j; + const swig_const_info *ci = 0; + const char *name = c + 10; + for (j = 0; const_table[j].type; ++j) { + if (strncmp(const_table[j].name, name, + strlen(const_table[j].name)) == 0) { + ci = &(const_table[j]); + break; + } + } + if (ci) { + void *ptr = (ci->type == SWIG_PY_POINTER) ? ci->pvalue : 0; + if (ptr) { + size_t shift = (ci->ptype) - types; + swig_type_info *ty = types_initial[shift]; + size_t ldoc = (c - methods[i].ml_doc); + size_t lptr = strlen(ty->name)+2*sizeof(void*)+2; + char *ndoc = (char*)malloc(ldoc + lptr + 10); + if (ndoc) { + char *buff = ndoc; + memcpy(buff, methods[i].ml_doc, ldoc); + buff += ldoc; + memcpy(buff, "swig_ptr: ", 10); + buff += 10; + SWIG_PackVoidPtr(buff, ptr, ty->name, lptr); + methods[i].ml_doc = ndoc; + } + } + } + } + } + } + +#ifdef __cplusplus +} +#endif + + + + +/* -----------------------------------------------------------------------------* + * Partial Init method + * -----------------------------------------------------------------------------*/ + +#ifdef __cplusplus +extern "C" +#endif + +SWIGEXPORT +#if PY_VERSION_HEX >= 0x03000000 +PyObject* +#else +void +#endif +SWIG_init(void) { + PyObject *m, *d, *md, *globals; + +#if PY_VERSION_HEX >= 0x03000000 + static struct PyModuleDef SWIG_module = { + PyModuleDef_HEAD_INIT, + SWIG_name, + NULL, + -1, + SwigMethods, + NULL, + NULL, + NULL, + NULL + }; +#endif + +#if defined(SWIGPYTHON_BUILTIN) + static SwigPyClientData SwigPyObject_clientdata = { + 0, 0, 0, 0, 0, 0, 0 + }; + static PyGetSetDef this_getset_def = { + (char *)"this", &SwigPyBuiltin_ThisClosure, NULL, NULL, NULL + }; + static SwigPyGetSet thisown_getset_closure = { + SwigPyObject_own, + SwigPyObject_own + }; + static PyGetSetDef thisown_getset_def = { + (char *)"thisown", SwigPyBuiltin_GetterClosure, SwigPyBuiltin_SetterClosure, NULL, &thisown_getset_closure + }; + PyTypeObject *builtin_pytype; + int builtin_base_count; + swig_type_info *builtin_basetype; + PyObject *tuple; + PyGetSetDescrObject *static_getset; + PyTypeObject *metatype; + PyTypeObject *swigpyobject; + SwigPyClientData *cd; + PyObject *public_interface, *public_symbol; + PyObject *this_descr; + PyObject *thisown_descr; + PyObject *self = 0; + int i; + + (void)builtin_pytype; + (void)builtin_base_count; + (void)builtin_basetype; + (void)tuple; + (void)static_getset; + (void)self; + + /* Metaclass is used to implement static member variables */ + metatype = SwigPyObjectType(); + assert(metatype); +#endif + + (void)globals; + + /* Create singletons now to avoid potential deadlocks with multi-threaded usage after module initialization */ + SWIG_This(); + SWIG_Python_TypeCache(); + SwigPyPacked_type(); +#ifndef SWIGPYTHON_BUILTIN + SwigPyObject_type(); +#endif + + /* Fix SwigMethods to carry the callback ptrs when needed */ + SWIG_Python_FixMethods(SwigMethods, swig_const_table, swig_types, swig_type_initial); + +#if PY_VERSION_HEX >= 0x03000000 + m = PyModule_Create(&SWIG_module); +#else + m = Py_InitModule(SWIG_name, SwigMethods); +#endif + + md = d = PyModule_GetDict(m); + (void)md; + + SWIG_InitializeModule(0); + +#ifdef SWIGPYTHON_BUILTIN + swigpyobject = SwigPyObject_TypeOnce(); + + SwigPyObject_stype = SWIG_MangledTypeQuery("_p_SwigPyObject"); + assert(SwigPyObject_stype); + cd = (SwigPyClientData*) SwigPyObject_stype->clientdata; + if (!cd) { + SwigPyObject_stype->clientdata = &SwigPyObject_clientdata; + SwigPyObject_clientdata.pytype = swigpyobject; + } else if (swigpyobject->tp_basicsize != cd->pytype->tp_basicsize) { + PyErr_SetString(PyExc_RuntimeError, "Import error: attempted to load two incompatible swig-generated modules."); +# if PY_VERSION_HEX >= 0x03000000 + return NULL; +# else + return; +# endif + } + + /* All objects have a 'this' attribute */ + this_descr = PyDescr_NewGetSet(SwigPyObject_type(), &this_getset_def); + (void)this_descr; + + /* All objects have a 'thisown' attribute */ + thisown_descr = PyDescr_NewGetSet(SwigPyObject_type(), &thisown_getset_def); + (void)thisown_descr; + + public_interface = PyList_New(0); + public_symbol = 0; + (void)public_symbol; + + PyDict_SetItemString(md, "__all__", public_interface); + SWIG_Py_DECREF(public_interface); + for (i = 0; SwigMethods[i].ml_name != NULL; ++i) + SwigPyBuiltin_AddPublicSymbol(public_interface, SwigMethods[i].ml_name); + for (i = 0; swig_const_table[i].name != 0; ++i) + SwigPyBuiltin_AddPublicSymbol(public_interface, swig_const_table[i].name); +#endif + + SWIG_InstallConstants(d,swig_const_table); + + +#ifdef Py_GIL_DISABLED + PyUnstable_Module_SetGIL(m, Py_MOD_GIL_NOT_USED); +#endif + +#if PY_VERSION_HEX >= 0x03000000 + return m; +#else + return; +#endif +} + diff --git a/lib/python3.12/site-packages/starlette-0.46.2.dist-info/INSTALLER b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/INSTALLER new file mode 100644 index 0000000000000000000000000000000000000000..a1b589e38a32041e49332e5e81c2d363dc418d68 --- /dev/null +++ b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/lib/python3.12/site-packages/starlette-0.46.2.dist-info/METADATA b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/METADATA new file mode 100644 index 0000000000000000000000000000000000000000..bb1935a27810ed970b6d2d668a2160f79d2c18cf --- /dev/null +++ b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/METADATA @@ -0,0 +1,176 @@ +Metadata-Version: 2.4 +Name: starlette +Version: 0.46.2 +Summary: The little ASGI library that shines. +Project-URL: Homepage, https://github.com/encode/starlette +Project-URL: Documentation, https://www.starlette.io/ +Project-URL: Changelog, https://www.starlette.io/release-notes/ +Project-URL: Funding, https://github.com/sponsors/encode +Project-URL: Source, https://github.com/encode/starlette +Author-email: Tom Christie +License-Expression: BSD-3-Clause +License-File: LICENSE.md +Classifier: Development Status :: 3 - Alpha +Classifier: Environment :: Web Environment +Classifier: Framework :: AnyIO +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Classifier: Programming Language :: Python :: 3.13 +Classifier: Topic :: Internet :: WWW/HTTP +Requires-Python: >=3.9 +Requires-Dist: anyio<5,>=3.6.2 +Requires-Dist: typing-extensions>=3.10.0; python_version < '3.10' +Provides-Extra: full +Requires-Dist: httpx<0.29.0,>=0.27.0; extra == 'full' +Requires-Dist: itsdangerous; extra == 'full' +Requires-Dist: jinja2; extra == 'full' +Requires-Dist: python-multipart>=0.0.18; extra == 'full' +Requires-Dist: pyyaml; extra == 'full' +Description-Content-Type: text/markdown + +

+ + + + starlette-logo + +

+ +

+ ✨ The little ASGI framework that shines. ✨ +

+ +--- + +[![Build Status](https://github.com/encode/starlette/workflows/Test%20Suite/badge.svg)](https://github.com/encode/starlette/actions) +[![Package version](https://badge.fury.io/py/starlette.svg)](https://pypi.python.org/pypi/starlette) +[![Supported Python Version](https://img.shields.io/pypi/pyversions/starlette.svg?color=%2334D058)](https://pypi.org/project/starlette) + +--- + +**Documentation**: https://www.starlette.io + +**Source Code**: https://github.com/encode/starlette + +--- + +# Starlette + +Starlette is a lightweight [ASGI][asgi] framework/toolkit, +which is ideal for building async web services in Python. + +It is production-ready, and gives you the following: + +* A lightweight, low-complexity HTTP web framework. +* WebSocket support. +* In-process background tasks. +* Startup and shutdown events. +* Test client built on `httpx`. +* CORS, GZip, Static Files, Streaming responses. +* Session and Cookie support. +* 100% test coverage. +* 100% type annotated codebase. +* Few hard dependencies. +* Compatible with `asyncio` and `trio` backends. +* Great overall performance [against independent benchmarks][techempower]. + +## Installation + +```shell +$ pip install starlette +``` + +You'll also want to install an ASGI server, such as [uvicorn](https://www.uvicorn.org/), [daphne](https://github.com/django/daphne/), or [hypercorn](https://hypercorn.readthedocs.io/en/latest/). + +```shell +$ pip install uvicorn +``` + +## Example + +```python title="main.py" +from starlette.applications import Starlette +from starlette.responses import JSONResponse +from starlette.routing import Route + + +async def homepage(request): + return JSONResponse({'hello': 'world'}) + +routes = [ + Route("/", endpoint=homepage) +] + +app = Starlette(debug=True, routes=routes) +``` + +Then run the application using Uvicorn: + +```shell +$ uvicorn main:app +``` + +## Dependencies + +Starlette only requires `anyio`, and the following are optional: + +* [`httpx`][httpx] - Required if you want to use the `TestClient`. +* [`jinja2`][jinja2] - Required if you want to use `Jinja2Templates`. +* [`python-multipart`][python-multipart] - Required if you want to support form parsing, with `request.form()`. +* [`itsdangerous`][itsdangerous] - Required for `SessionMiddleware` support. +* [`pyyaml`][pyyaml] - Required for `SchemaGenerator` support. + +You can install all of these with `pip install starlette[full]`. + +## Framework or Toolkit + +Starlette is designed to be used either as a complete framework, or as +an ASGI toolkit. You can use any of its components independently. + +```python +from starlette.responses import PlainTextResponse + + +async def app(scope, receive, send): + assert scope['type'] == 'http' + response = PlainTextResponse('Hello, world!') + await response(scope, receive, send) +``` + +Run the `app` application in `example.py`: + +```shell +$ uvicorn example:app +INFO: Started server process [11509] +INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) +``` + +Run uvicorn with `--reload` to enable auto-reloading on code changes. + +## Modularity + +The modularity that Starlette is designed on promotes building re-usable +components that can be shared between any ASGI framework. This should enable +an ecosystem of shared middleware and mountable applications. + +The clean API separation also means it's easier to understand each component +in isolation. + +--- + +

Starlette is BSD licensed code.
Designed & crafted with care.
— ⭐️ —

+ +[asgi]: https://asgi.readthedocs.io/en/latest/ +[httpx]: https://www.python-httpx.org/ +[jinja2]: https://jinja.palletsprojects.com/ +[python-multipart]: https://multipart.fastapiexpert.com/ +[itsdangerous]: https://itsdangerous.palletsprojects.com/ +[sqlalchemy]: https://www.sqlalchemy.org +[pyyaml]: https://pyyaml.org/wiki/PyYAMLDocumentation +[techempower]: https://www.techempower.com/benchmarks/#hw=ph&test=fortune&l=zijzen-sf diff --git a/lib/python3.12/site-packages/starlette-0.46.2.dist-info/RECORD b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/RECORD new file mode 100644 index 0000000000000000000000000000000000000000..6cd8259f776ec334bc30bf19540cc4a31d0689b5 --- /dev/null +++ b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/RECORD @@ -0,0 +1,74 @@ +starlette-0.46.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +starlette-0.46.2.dist-info/METADATA,sha256=ZQ65swcGWuW1-L0UrDKo2SaF90Jux18XK9OmZBmtEyg,6167 +starlette-0.46.2.dist-info/RECORD,, +starlette-0.46.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 +starlette-0.46.2.dist-info/licenses/LICENSE.md,sha256=3LlWd6AiQCQxh-lk-UGEfRmxeCHPmeWvrmhPqzKMGb8,1518 +starlette/__init__.py,sha256=lWEdbG8IshPF1zUeQxPyfsi3uFhuXQmN7phOE5OaeI8,23 +starlette/__pycache__/__init__.cpython-312.pyc,, +starlette/__pycache__/_exception_handler.cpython-312.pyc,, +starlette/__pycache__/_utils.cpython-312.pyc,, +starlette/__pycache__/applications.cpython-312.pyc,, +starlette/__pycache__/authentication.cpython-312.pyc,, +starlette/__pycache__/background.cpython-312.pyc,, +starlette/__pycache__/concurrency.cpython-312.pyc,, +starlette/__pycache__/config.cpython-312.pyc,, +starlette/__pycache__/convertors.cpython-312.pyc,, +starlette/__pycache__/datastructures.cpython-312.pyc,, +starlette/__pycache__/endpoints.cpython-312.pyc,, +starlette/__pycache__/exceptions.cpython-312.pyc,, +starlette/__pycache__/formparsers.cpython-312.pyc,, +starlette/__pycache__/requests.cpython-312.pyc,, +starlette/__pycache__/responses.cpython-312.pyc,, +starlette/__pycache__/routing.cpython-312.pyc,, +starlette/__pycache__/schemas.cpython-312.pyc,, +starlette/__pycache__/staticfiles.cpython-312.pyc,, +starlette/__pycache__/status.cpython-312.pyc,, +starlette/__pycache__/templating.cpython-312.pyc,, +starlette/__pycache__/testclient.cpython-312.pyc,, +starlette/__pycache__/types.cpython-312.pyc,, +starlette/__pycache__/websockets.cpython-312.pyc,, +starlette/_exception_handler.py,sha256=OM4H48d5iyv8_ofxBDYy1PflsJGlr40jFZPI3DYZTQg,2219 +starlette/_utils.py,sha256=Lg65uXpJ3Tq0lqik0t-A6nQDNhxMl9NttI7fjxY0hxM,2764 +starlette/applications.py,sha256=hNiaF39WBGRPt8Ht6Zw67XJ5B1CzCUj2cC-8KIZGddk,10678 +starlette/authentication.py,sha256=9dlyeykDMhoxgmI9hFVZ38uu8Nl6k2NZiaoa607Z8hc,4948 +starlette/background.py,sha256=50zBQ_50lV9YGRGevBUFQ4DOwisU47pUV_PfiBG9Cg4,1257 +starlette/concurrency.py,sha256=zjBYz9UPaXpUKx1qVzocVQHlxDDi90wOOUKubXjXWTI,1746 +starlette/config.py,sha256=xCXgQTgMaeuyXaHRFor7EJiAVMUdjnu6Hgwz4rr6sUU,4445 +starlette/convertors.py,sha256=RBd7q6UynlJl4xP_MXDrRmtH8N55re8d_SQaztmwPQw,2302 +starlette/datastructures.py,sha256=7IkoXZ5b0EaieymYa46Sncwd7GVAnyJ5RI-LQvKLFx0,22262 +starlette/endpoints.py,sha256=RgFXHAw8_Ktgqea5T5e4jEWY73LbB822odx6LXI-_6I,5098 +starlette/exceptions.py,sha256=tIphlZa8EsQfKw3-xw5J3ZN1GjaR4UcxfJK69Ad2hG8,1066 +starlette/formparsers.py,sha256=GxhueF4HrJeLJDFGpPFm5WZn7C90VBGS2Faz90JP1gY,11045 +starlette/middleware/__init__.py,sha256=3WljcfADnSltJrVUuFgpvJiZKcjsjC1Ih9aqYUvSknk,1224 +starlette/middleware/__pycache__/__init__.cpython-312.pyc,, +starlette/middleware/__pycache__/authentication.cpython-312.pyc,, +starlette/middleware/__pycache__/base.cpython-312.pyc,, +starlette/middleware/__pycache__/cors.cpython-312.pyc,, +starlette/middleware/__pycache__/errors.cpython-312.pyc,, +starlette/middleware/__pycache__/exceptions.cpython-312.pyc,, +starlette/middleware/__pycache__/gzip.cpython-312.pyc,, +starlette/middleware/__pycache__/httpsredirect.cpython-312.pyc,, +starlette/middleware/__pycache__/sessions.cpython-312.pyc,, +starlette/middleware/__pycache__/trustedhost.cpython-312.pyc,, +starlette/middleware/__pycache__/wsgi.cpython-312.pyc,, +starlette/middleware/authentication.py,sha256=ftWVVbETxrr0gOwtxwULww362Wkt1s9iOVu4s1LJ53I,1791 +starlette/middleware/base.py,sha256=O8u_zoxndbFGaykG5peYd42koxjp_mYDk76OKVuCYO8,8971 +starlette/middleware/cors.py,sha256=-ih4JxmeRaKhmv076DcJE-wd0K2FJbo23gzZdLrJb00,7051 +starlette/middleware/errors.py,sha256=bomTUfQIo7CIwnZQ8fFz4BCDY5idQnrsrhGSpZNiH-s,8066 +starlette/middleware/exceptions.py,sha256=Q96acXz9bRIUZORAKjlsbcaFuI0-m61cSXmfFq_ND60,2791 +starlette/middleware/gzip.py,sha256=VaNOHZleGS_YWueBFVFZ-RboWQmcQI8hjgZDKLVklOM,5697 +starlette/middleware/httpsredirect.py,sha256=SNTleaYALGoITV7xwbic4gB6VYdM8Ylea_ykciUz31g,848 +starlette/middleware/sessions.py,sha256=Cpbql_BTanVqDbhPGdoUrNvHIKf9D3Yuc48cJZerFTU,3566 +starlette/middleware/trustedhost.py,sha256=dXbW22C3RYNp3TsdfCuE0MNS-VZFcN8B_S3wYrzETm8,2203 +starlette/middleware/wsgi.py,sha256=0vLstcalV-kq2YyqQR9NCLiPNgokW7nDvyRqQpj_5iQ,5386 +starlette/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +starlette/requests.py,sha256=X_rmfl_pgnndvqk0Kxd_bCjxIe_PUO0N64DJMpeOUBE,11693 +starlette/responses.py,sha256=_9PWDCpoqZzT9iVTwLNuQ47RuMTzp6NbSknaW4lpJwM,20226 +starlette/routing.py,sha256=mKt5XcuSBg_08__XdXp5WVh3qCJMPLMaYfe1bXFzI8w,34550 +starlette/schemas.py,sha256=kKVRFG_kicD8geEnlfHoEsBCE-8MsdXyPE0VM4f2Q24,5181 +starlette/staticfiles.py,sha256=pmxmrueM67-X16sYAVHEIu61BJr221XNfq5d1UbFls4,8474 +starlette/status.py,sha256=e70xV6wYFR5bdmkYkgYCSwZk1L2FdKDwA6u4zCjmypQ,2820 +starlette/templating.py,sha256=HzG3S0THU-r7q-1AqRZyaxLJOwVyYAcxwvzIwU7tQZ0,8408 +starlette/testclient.py,sha256=G0Db7thEEaE6unArj34D_qG3NT_OI_1Io-3P4z_psFM,27987 +starlette/types.py,sha256=I2yIP5R1qyVD1GBiJZQ_I9dprcYUOVVK8fHa6UQVsqg,1048 +starlette/websockets.py,sha256=M_8gmQmthBHvqdVfLIzs4yuQpup7B853tPSaNQB_UMo,8332 diff --git a/lib/python3.12/site-packages/starlette-0.46.2.dist-info/WHEEL b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/WHEEL new file mode 100644 index 0000000000000000000000000000000000000000..12228d414b6cfed7c39d3781c85c63256a1d7fb5 --- /dev/null +++ b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/WHEEL @@ -0,0 +1,4 @@ +Wheel-Version: 1.0 +Generator: hatchling 1.27.0 +Root-Is-Purelib: true +Tag: py3-none-any diff --git a/lib/python3.12/site-packages/starlette-0.46.2.dist-info/licenses/LICENSE.md b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/licenses/LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..d16a60ec5b9963ef86b35a52ac92227014618e6c --- /dev/null +++ b/lib/python3.12/site-packages/starlette-0.46.2.dist-info/licenses/LICENSE.md @@ -0,0 +1,27 @@ +Copyright © 2018, [Encode OSS Ltd](https://www.encode.io/). +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/lib/python3.12/site-packages/tensorboard/__init__.py b/lib/python3.12/site-packages/tensorboard/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..95bc174e64d652d19d4353a28dc1c51858f922b9 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/__init__.py @@ -0,0 +1,113 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorBoard is a webapp for understanding TensorFlow runs and graphs.""" + + +from tensorboard import lazy as _lazy +from tensorboard import version as _version + +# TensorBoard public API. +__all__ = [ + "__version__", + "errors", + "notebook", + "program", + "summary", +] + + +# Please be careful when changing the structure of this file. +# +# The lazy imports in this file must use `importlib.import_module`, not +# `import tensorboard.foo` or `from tensorboard import foo`, or it will +# be impossible to reload the TensorBoard module without breaking these +# top-level public APIs. This has to do with the gory details of +# Python's module system. Take `tensorboard.notebook` as an example: +# +# - When the `tensorboard` module (that's us!) is initialized, its +# `notebook` attribute is initialized to a new LazyModule. The +# actual `tensorboard.notebook` submodule is not loaded. +# +# - When the `tensorboard.notebook` submodule is first loaded, Python +# _reassigns_ the `notebook` attribute on the `tensorboard` module +# object to point to the underlying `tensorboard.notebook` module +# object, rather than its former LazyModule value. This occurs +# whether the module is loaded via the lazy module or directly as an +# import: +# +# - import tensorboard; tensorboard.notebook.start(...) # one way +# - from tensorboard import notebook # other way; same effect +# +# - When the `tensorboard` module is reloaded, its `notebook` +# attribute is once again bound to a (new) LazyModule, while the +# `tensorboard.notebook` module object is unaffected and still +# exists in `sys.modules`. But then... +# +# - When the new LazyModule is forced, it must resolve to the existing +# `tensorboard.notebook` module object rather than itself (which +# just creates a stack overflow). If the LazyModule load function +# uses `import tensorboard.notebook; return tensorboard.notebook`, +# then the first statement will do _nothing_ because the +# `tensorboard.notebook` module is already loaded, and the second +# statement will return the LazyModule itself. The same goes for the +# `from tensorboard import notebook` form. We need to ensure that +# the submodule is loaded and then pull the actual module object out +# of `sys.modules`... which is exactly what `importlib` handles for +# us. +# +# See for +# additional discussion. + + +@_lazy.lazy_load("tensorboard.errors") +def errors(): + import importlib + + return importlib.import_module("tensorboard.errors") + + +@_lazy.lazy_load("tensorboard.notebook") +def notebook(): + import importlib + + return importlib.import_module("tensorboard.notebook") + + +@_lazy.lazy_load("tensorboard.program") +def program(): + import importlib + + return importlib.import_module("tensorboard.program") + + +@_lazy.lazy_load("tensorboard.summary") +def summary(): + import importlib + + return importlib.import_module("tensorboard.summary") + + +def load_ipython_extension(ipython): + """IPython API entry point. + + Only intended to be called by the IPython runtime. + + See: + https://ipython.readthedocs.io/en/stable/config/extensions/index.html + """ + notebook._load_ipython_extension(ipython) + + +__version__ = _version.VERSION diff --git a/lib/python3.12/site-packages/tensorboard/assets.py b/lib/python3.12/site-packages/tensorboard/assets.py new file mode 100644 index 0000000000000000000000000000000000000000..e9ecf3cb6b95a112744d668babddf19179e4571c --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/assets.py @@ -0,0 +1,36 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Bindings for TensorBoard frontend assets.""" + +import os + +from tensorboard.util import tb_logging + +logger = tb_logging.get_logger() + + +def get_default_assets_zip_provider(): + """Try to get a function to provide frontend assets. + + Returns: + Either (a) a callable that takes no arguments and returns an open + file handle to a Zip archive of frontend assets, or (b) `None`, if + the frontend assets cannot be found. + """ + path = os.path.join(os.path.dirname(__file__), "webfiles.zip") + if not os.path.exists(path): + logger.warning("webfiles.zip static assets not found: %s", path) + return None + return lambda: open(path, "rb") diff --git a/lib/python3.12/site-packages/tensorboard/auth.py b/lib/python3.12/site-packages/tensorboard/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..48611d40451b299719b954decd0197217335a043 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/auth.py @@ -0,0 +1,102 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Experimental framework for authentication in TensorBoard.""" + +import abc + + +class AuthProvider(metaclass=abc.ABCMeta): + """Authentication provider for a specific kind of credential.""" + + def authenticate(self, environ): + """Produce an opaque auth token from a WSGI request environment. + + Args: + environ: A WSGI environment `dict`; see PEP 3333. + + Returns: + A Python object representing an auth token. The representation + and semantics depend on the particular `AuthProvider` + implementation. + + Raises: + Exception: Any error, usually `tensorboard.errors.PublicError` + subclasses (like `PermissionDenied`) but also possibly a + custom error type that should propagate to a WSGI middleware + for effecting a redirect-driven auth flow. + """ + pass + + +class AuthContext: + """Authentication context within the scope of a single request. + + Auth providers are keyed within an `AuthContext` by arbitrary + unique keys. It may often make sense for the key used for an + auth provider to simply be that provider's type object. + """ + + def __init__(self, providers, environ): + """Create an auth context. + + Args: + providers: A mapping from provider keys (opaque values) to + `AuthProvider` implementations. + environ: A WSGI environment (see PEP 3333). + """ + self._environ = environ + self._providers = providers + self._cache = {} + + @classmethod + def empty(cls): + """Create an auth context with no registered providers. + + Returns: + A new `AuthContext` value for which any call to `get` will + fail with a `KeyError`. + """ + # Use an empty dict for the environ. This is not a valid WSGI + # environment, but it doesn't matter because it's never used. + return cls({}, {}) + + def get(self, provider_key): + """Get an auth token from the auth provider with the given key. + + If successful, the result will be cached on this auth context. + If unsuccessful, nothing will be cached, so a future call will + invoke the underlying `AuthProvider.authenticate` method again. + + This method is not thread-safe. If multiple threads share an + auth context for a single request, then they must synchronize + externally when calling this method. + + Returns: + The result of `provider.authenticate(...)` for the auth + provider specified by `provider_key`. + + Raises: + KeyError: If the given `provider_key` does not correspond to + any registered `AuthProvider`. + Exception: As raised by the underlying `AuthProvider`. + """ + provider = self._providers[provider_key] + sentinel = object() + value = self._cache.get(provider_key, sentinel) + if value is not sentinel: + return value + value = provider.authenticate(self._environ) + self._cache[provider_key] = value + return value diff --git a/lib/python3.12/site-packages/tensorboard/context.py b/lib/python3.12/site-packages/tensorboard/context.py new file mode 100644 index 0000000000000000000000000000000000000000..2631f1d51e958cf8aa1a19e11ee69d94b35ff27f --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/context.py @@ -0,0 +1,130 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Request-scoped context.""" + +from tensorboard import auth as auth_lib + + +# A `RequestContext` value is stored on WSGI environments under this key. +_WSGI_KEY = "tensorboard.request_context" + + +class RequestContext: + """Container of request-scoped values. + + This context is for cross-cutting concerns: authentication, + authorization, auditing, internationalization, logging, and so on. + It is not simply for passing commonly used parameters to functions. + + `RequestContext` values are to be treated as immutable. + + Fields: + auth: An `AuthContext`, which may be empty but is never `None`. + remote_ip: An `ipaddress.IPv4Address` or `ipaddress.IPv6Address` or None. + Best guess of the IP Address of the end user. + x_forwarded_for: A tuple of `ipaddress.IPv4Address` or `ipaddress.IPv6Address`, + which may be empty but is never None. This should be parsed value of X-Forwarded-For + HTTP header from the request. + client_feature_flags: A dict of string to arbitrary type. These represent + feature flag key/value pairs sent by the client application. Usage of + client_feature_flags should know the name of the feature flag key and + should know and validate the type of the value. + """ + + def __init__( + self, + auth=None, + remote_ip=None, + x_forwarded_for=None, + client_feature_flags=None, + ): + """Create a request context. + + The argument list is sorted and may be extended in the future; + therefore, callers must pass only named arguments to this + initializer. + + Args: + See "Fields" on class docstring. All arguments are optional + and will be replaced with default values if appropriate. + """ + self._auth = auth if auth is not None else auth_lib.AuthContext.empty() + self._remote_ip = remote_ip + self._x_forwarded_for = x_forwarded_for or () + self._client_feature_flags = client_feature_flags or {} + + @property + def auth(self): + return self._auth + + @property + def remote_ip(self): + return self._remote_ip + + @property + def x_forwarded_for(self): + return self._x_forwarded_for + + @property + def client_feature_flags(self): + return self._client_feature_flags + + def replace(self, **kwargs): + """Create a copy of this context with updated key-value pairs. + + Analogous to `namedtuple._replace`. For example, to create a new + request context like `ctx` but with auth context `auth`, call + `ctx.replace(auth=auth)`. + + Args: + As to `__init__`. + + Returns: + A new context like this one but with the specified updates. + """ + kwargs.setdefault("auth", self.auth) + kwargs.setdefault("remote_ip", self.remote_ip) + kwargs.setdefault("x_forwarded_for", self.x_forwarded_for) + kwargs.setdefault("client_feature_flags", self.client_feature_flags) + return type(self)(**kwargs) + + +def from_environ(environ): + """Get a `RequestContext` from a WSGI environment. + + See also `set_in_environ`. + + Args: + environ: A WSGI environment (see PEP 3333). + + Returns: + The `RequestContext` stored in the WSGI environment, or an empty + `RequestContext` if none is stored. + """ + result = environ.get(_WSGI_KEY) + return result if result is not None else RequestContext() + + +def set_in_environ(environ, ctx): + """Set the `RequestContext` in a WSGI environment. + + After `set_in_environ(e, ctx)`, `from_environ(e) is ctx`. The input + environment is mutated. + + Args: + environ: A WSGI environment to update. + ctx: A new `RequestContext` value. + """ + environ[_WSGI_KEY] = ctx diff --git a/lib/python3.12/site-packages/tensorboard/data_compat.py b/lib/python3.12/site-packages/tensorboard/data_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..3a53ead3f24b4bba7bd0540287af6e9f96fdd0f9 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/data_compat.py @@ -0,0 +1,160 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities to migrate legacy protos to their modern equivalents.""" + + +import numpy as np + +from tensorboard.compat.proto import event_pb2 +from tensorboard.compat.proto import summary_pb2 +from tensorboard.plugins.audio import metadata as audio_metadata +from tensorboard.plugins.histogram import metadata as histogram_metadata +from tensorboard.plugins.image import metadata as image_metadata +from tensorboard.plugins.scalar import metadata as scalar_metadata +from tensorboard.util import tensor_util + + +def migrate_event(event): + if not event.HasField("summary"): + return event + old_values = event.summary.value + new_values = [migrate_value(value) for value in old_values] + # Optimization: Don't create a new event if there were no changes. + if len(old_values) == len(new_values) and all( + x is y for (x, y) in zip(old_values, new_values) + ): + return event + result = event_pb2.Event() + result.CopyFrom(event) + del result.summary.value[:] + result.summary.value.extend(new_values) + return result + + +def migrate_value(value): + """Convert `value` to a new-style value, if necessary and possible. + + An "old-style" value is a value that uses any `value` field other than + the `tensor` field. A "new-style" value is a value that uses the + `tensor` field. TensorBoard continues to support old-style values on + disk; this method converts them to new-style values so that further + code need only deal with one data format. + + Arguments: + value: A `Summary.Value` object. This argument is not modified. + + Returns: + If the `value` is an old-style value for which there is a new-style + equivalent, the result is the new-style value. Otherwise---if the + value is already new-style or does not yet have a new-style + equivalent---the value will be returned unchanged. + + :type value: Summary.Value + :rtype: Summary.Value + """ + handler = { + "histo": _migrate_histogram_value, + "image": _migrate_image_value, + "audio": _migrate_audio_value, + "simple_value": _migrate_scalar_value, + }.get(value.WhichOneof("value")) + return handler(value) if handler else value + + +def make_summary(tag, metadata, data): + tensor_proto = tensor_util.make_tensor_proto(data) + return summary_pb2.Summary.Value( + tag=tag, metadata=metadata, tensor=tensor_proto + ) + + +def _migrate_histogram_value(value): + """Convert `old-style` histogram value to `new-style`. + + The "old-style" format can have outermost bucket limits of -DBL_MAX and + DBL_MAX, which are problematic for visualization. We replace those here + with the actual min and max values seen in the input data, but then in + order to avoid introducing "backwards" buckets (where left edge > right + edge), we first must drop all empty buckets on the left and right ends. + """ + histogram_value = value.histo + bucket_counts = histogram_value.bucket + # Find the indices of the leftmost and rightmost non-empty buckets. + n = len(bucket_counts) + start = next((i for i in range(n) if bucket_counts[i] > 0), n) + end = next((i for i in reversed(range(n)) if bucket_counts[i] > 0), -1) + if start > end: + # If all input buckets were empty, treat it as a zero-bucket + # new-style histogram. + buckets = np.zeros([0, 3], dtype=np.float32) + else: + # Discard empty buckets on both ends, and keep only the "inner" + # edges from the remaining buckets. Note that bucket indices range + # from `start` to `end` inclusive, but bucket_limit indices are + # exclusive of `end` - this is because bucket_limit[i] is the + # right-hand edge for bucket[i]. + bucket_counts = bucket_counts[start : end + 1] + inner_edges = histogram_value.bucket_limit[start:end] + # Use min as the left-hand limit for the first non-empty bucket. + bucket_lefts = [histogram_value.min] + inner_edges + # Use max as the right-hand limit for the last non-empty bucket. + bucket_rights = inner_edges + [histogram_value.max] + buckets = np.array( + [bucket_lefts, bucket_rights, bucket_counts], dtype=np.float32 + ).transpose() + + summary_metadata = histogram_metadata.create_summary_metadata( + display_name=value.metadata.display_name or value.tag, + description=value.metadata.summary_description, + ) + + return make_summary(value.tag, summary_metadata, buckets) + + +def _migrate_image_value(value): + image_value = value.image + data = [ + str(image_value.width).encode("ascii"), + str(image_value.height).encode("ascii"), + image_value.encoded_image_string, + ] + + summary_metadata = image_metadata.create_summary_metadata( + display_name=value.metadata.display_name or value.tag, + description=value.metadata.summary_description, + converted_to_tensor=True, + ) + return make_summary(value.tag, summary_metadata, data) + + +def _migrate_audio_value(value): + audio_value = value.audio + data = [[audio_value.encoded_audio_string, b""]] # empty label + summary_metadata = audio_metadata.create_summary_metadata( + display_name=value.metadata.display_name or value.tag, + description=value.metadata.summary_description, + encoding=audio_metadata.Encoding.Value("WAV"), + converted_to_tensor=True, + ) + return make_summary(value.tag, summary_metadata, data) + + +def _migrate_scalar_value(value): + scalar_value = value.simple_value + summary_metadata = scalar_metadata.create_summary_metadata( + display_name=value.metadata.display_name or value.tag, + description=value.metadata.summary_description, + ) + return make_summary(value.tag, summary_metadata, scalar_value) diff --git a/lib/python3.12/site-packages/tensorboard/dataclass_compat.py b/lib/python3.12/site-packages/tensorboard/dataclass_compat.py new file mode 100644 index 0000000000000000000000000000000000000000..f2f3a744349c9527fd0eb081ca398fe0208b94b5 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/dataclass_compat.py @@ -0,0 +1,225 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities to migrate legacy summaries/events to generic data form. + +For legacy summaries, this populates the `SummaryMetadata.data_class` +field and makes any necessary transformations to the tensor value. For +`graph_def` events, this creates a new summary event. + +This should be effected after the `data_compat` transformation. +""" + + +from tensorboard.compat.proto import event_pb2 +from tensorboard.compat.proto import summary_pb2 +from tensorboard.plugins.audio import metadata as audio_metadata +from tensorboard.plugins.custom_scalar import ( + metadata as custom_scalars_metadata, +) +from tensorboard.plugins.graph import metadata as graphs_metadata +from tensorboard.plugins.histogram import metadata as histograms_metadata +from tensorboard.plugins.hparams import metadata as hparams_metadata +from tensorboard.plugins.image import metadata as images_metadata +from tensorboard.plugins.mesh import metadata as mesh_metadata +from tensorboard.plugins.pr_curve import metadata as pr_curves_metadata +from tensorboard.plugins.scalar import metadata as scalars_metadata +from tensorboard.plugins.text import metadata as text_metadata +from tensorboard.util import tensor_util + + +def migrate_event(event, initial_metadata): + """Migrate an event to a sequence of events. + + Args: + event: An `event_pb2.Event`. The caller transfers ownership of the + event to this method; the event may be mutated, and may or may + not appear in the returned sequence. + initial_metadata: Map from tag name (string) to `SummaryMetadata` + proto for the initial occurrence of the given tag within the + enclosing run. While loading a given run, the caller should + always pass the same dictionary here, initially `{}`; this + function will mutate it and reuse it for future calls. + + Returns: + A sequence of `event_pb2.Event`s to use instead of `event`. + """ + what = event.WhichOneof("what") + if what == "graph_def": + return _migrate_graph_event(event) + if what == "tagged_run_metadata": + return _migrate_tagged_run_metadata_event(event) + if what == "summary": + return _migrate_summary_event(event, initial_metadata) + return (event,) + + +def _migrate_graph_event(old_event): + result = event_pb2.Event() + result.wall_time = old_event.wall_time + result.step = old_event.step + value = result.summary.value.add(tag=graphs_metadata.RUN_GRAPH_NAME) + graph_bytes = old_event.graph_def + value.tensor.CopyFrom(tensor_util.make_tensor_proto([graph_bytes])) + value.metadata.plugin_data.plugin_name = graphs_metadata.PLUGIN_NAME + # `value.metadata.plugin_data.content` left empty + value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE + # As long as the graphs plugin still reads the old format, keep both + # the old event and the new event to maintain compatibility. + return (old_event, result) + + +def _migrate_tagged_run_metadata_event(old_event): + result = event_pb2.Event() + result.wall_time = old_event.wall_time + result.step = old_event.step + trm = old_event.tagged_run_metadata + value = result.summary.value.add(tag=trm.tag) + value.tensor.CopyFrom(tensor_util.make_tensor_proto([trm.run_metadata])) + value.metadata.plugin_data.plugin_name = ( + graphs_metadata.PLUGIN_NAME_TAGGED_RUN_METADATA + ) + # `value.metadata.plugin_data.content` left empty + value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE + return (result,) + + +def _migrate_summary_event(event, initial_metadata): + values = event.summary.value + new_values = [ + new for old in values for new in _migrate_value(old, initial_metadata) + ] + # Optimization: Don't create a new event if there were no shallow + # changes (there may still have been in-place changes). + if len(values) == len(new_values) and all( + x is y for (x, y) in zip(values, new_values) + ): + return (event,) + del event.summary.value[:] + event.summary.value.extend(new_values) + return (event,) + + +def _migrate_value(value, initial_metadata): + """Convert an old value to a stream of new values. May mutate.""" + metadata = initial_metadata.get(value.tag) + initial = False + if metadata is None: + initial = True + # Retain a copy of the initial metadata, so that even after we + # update its data class we know whether to also transform later + # events in this time series. + metadata = summary_pb2.SummaryMetadata() + metadata.CopyFrom(value.metadata) + initial_metadata[value.tag] = metadata + if metadata.data_class != summary_pb2.DATA_CLASS_UNKNOWN: + return (value,) + plugin_name = metadata.plugin_data.plugin_name + if plugin_name == histograms_metadata.PLUGIN_NAME: + return _migrate_histogram_value(value) + if plugin_name == images_metadata.PLUGIN_NAME: + return _migrate_image_value(value) + if plugin_name == audio_metadata.PLUGIN_NAME: + return _migrate_audio_value(value) + if plugin_name == scalars_metadata.PLUGIN_NAME: + return _migrate_scalar_value(value) + if plugin_name == text_metadata.PLUGIN_NAME: + return _migrate_text_value(value) + if plugin_name == hparams_metadata.PLUGIN_NAME: + return _migrate_hparams_value(value) + if plugin_name == pr_curves_metadata.PLUGIN_NAME: + return _migrate_pr_curve_value(value) + if plugin_name == mesh_metadata.PLUGIN_NAME: + return _migrate_mesh_value(value) + if plugin_name == custom_scalars_metadata.PLUGIN_NAME: + return _migrate_custom_scalars_value(value) + if plugin_name in [ + graphs_metadata.PLUGIN_NAME_RUN_METADATA, + graphs_metadata.PLUGIN_NAME_RUN_METADATA_WITH_GRAPH, + graphs_metadata.PLUGIN_NAME_KERAS_MODEL, + ]: + return _migrate_graph_sub_plugin_value(value) + return (value,) + + +def _migrate_scalar_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_SCALAR + return (value,) + + +def _migrate_histogram_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_TENSOR + return (value,) + + +def _migrate_image_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE + return (value,) + + +def _migrate_text_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_TENSOR + return (value,) + + +def _migrate_audio_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE + tensor = value.tensor + # Project out just the first axis: actual audio clips. + stride = 1 + while len(tensor.tensor_shape.dim) > 1: + stride *= tensor.tensor_shape.dim.pop().size + if stride != 1: + tensor.string_val[:] = tensor.string_val[::stride] + return (value,) + + +def _migrate_hparams_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_TENSOR + if not value.HasField("tensor"): + value.tensor.CopyFrom(hparams_metadata.NULL_TENSOR) + return (value,) + + +def _migrate_pr_curve_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_TENSOR + return (value,) + + +def _migrate_mesh_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_TENSOR + return (value,) + + +def _migrate_custom_scalars_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_TENSOR + return (value,) + + +def _migrate_graph_sub_plugin_value(value): + if value.HasField("metadata"): + value.metadata.data_class = summary_pb2.DATA_CLASS_BLOB_SEQUENCE + shape = value.tensor.tensor_shape.dim + if not shape: + shape.add(size=1) + return (value,) diff --git a/lib/python3.12/site-packages/tensorboard/default.py b/lib/python3.12/site-packages/tensorboard/default.py new file mode 100644 index 0000000000000000000000000000000000000000..7e6711337af76d91b9d8cb28f6f1f9ba4252269b --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/default.py @@ -0,0 +1,126 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Collection of first-party plugins. + +This module exists to isolate tensorboard.program from the potentially +heavyweight build dependencies for first-party plugins. This way people +doing custom builds of TensorBoard have the option to only pay for the +dependencies they want. + +This module also grants the flexibility to those doing custom builds, to +automatically inherit the centrally-maintained list of standard plugins, +for less repetition. +""" + + +import logging + +import pkg_resources + +from tensorboard.plugins.audio import audio_plugin +from tensorboard.plugins.core import core_plugin +from tensorboard.plugins.custom_scalar import custom_scalars_plugin +from tensorboard.plugins.debugger_v2 import debugger_v2_plugin +from tensorboard.plugins.distribution import distributions_plugin +from tensorboard.plugins.graph import graphs_plugin +from tensorboard.plugins.histogram import histograms_plugin +from tensorboard.plugins.hparams import hparams_plugin +from tensorboard.plugins.image import images_plugin +from tensorboard.plugins.metrics import metrics_plugin +from tensorboard.plugins.pr_curve import pr_curves_plugin +from tensorboard.plugins.profile_redirect import profile_redirect_plugin +from tensorboard.plugins.scalar import scalars_plugin +from tensorboard.plugins.text import text_plugin +from tensorboard.plugins.mesh import mesh_plugin +from tensorboard.plugins.wit_redirect import wit_redirect_plugin + + +logger = logging.getLogger(__name__) + + +# Ordering matters. The order in which these lines appear determines the +# ordering of tabs in TensorBoard's GUI. +_PLUGINS = [ + core_plugin.CorePluginLoader(include_debug_info=True), + metrics_plugin.MetricsPlugin, + scalars_plugin.ScalarsPlugin, + custom_scalars_plugin.CustomScalarsPlugin, + images_plugin.ImagesPlugin, + audio_plugin.AudioPlugin, + debugger_v2_plugin.DebuggerV2Plugin, + graphs_plugin.GraphsPlugin, + distributions_plugin.DistributionsPlugin, + histograms_plugin.HistogramsPlugin, + text_plugin.TextPlugin, + pr_curves_plugin.PrCurvesPlugin, + profile_redirect_plugin.ProfileRedirectPluginLoader, + hparams_plugin.HParamsPlugin, + mesh_plugin.MeshPlugin, + wit_redirect_plugin.WITRedirectPluginLoader, +] + + +def get_plugins(): + """Returns a list specifying all known TensorBoard plugins. + + This includes both first-party, statically bundled plugins and + dynamic plugins. + + This list can be passed to the `tensorboard.program.TensorBoard` API. + + Returns: + The list of default first-party plugins. + """ + return get_static_plugins() + get_dynamic_plugins() + + +def get_static_plugins(): + """Returns a list specifying TensorBoard's default first-party plugins. + + Plugins are specified in this list either via a TBLoader instance to load the + plugin, or the TBPlugin class itself which will be loaded using a BasicLoader. + + This list can be passed to the `tensorboard.program.TensorBoard` API. + + Returns: + The list of default first-party plugins. + + :rtype: list[Type[base_plugin.TBLoader] | Type[base_plugin.TBPlugin]] + """ + + return _PLUGINS[:] + + +def get_dynamic_plugins(): + """Returns a list specifying TensorBoard's dynamically loaded plugins. + + A dynamic TensorBoard plugin is specified using entry_points [1] and it is + the robust way to integrate plugins into TensorBoard. + + This list can be passed to the `tensorboard.program.TensorBoard` API. + + Returns: + The list of dynamic plugins. + + :rtype: list[Type[base_plugin.TBLoader] | Type[base_plugin.TBPlugin]] + + [1]: https://packaging.python.org/specifications/entry-points/ + """ + return [ + entry_point.resolve() + for entry_point in pkg_resources.iter_entry_points( + "tensorboard_plugins" + ) + ] diff --git a/lib/python3.12/site-packages/tensorboard/errors.py b/lib/python3.12/site-packages/tensorboard/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..1951bc9260e2bf7d1476b51acea383ec9d686cec --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/errors.py @@ -0,0 +1,130 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Error codes (experimental module). + +These types represent a subset of the Google error codes [1], which are +also used by TensorFlow, gRPC, et al. + +When an HTTP handler raises one of these errors, the TensorBoard core +application will catch it and automatically serve a properly formatted +response with the error message. + +[1]: https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto +""" + + +class PublicError(RuntimeError): + """An error whose text does not contain sensitive information. + + Fields: + http_code: Integer between 400 and 599 inclusive (e.g., 404). + headers: List of additional key-value pairs to include in the + response body, like `[("Allow", "GET")]` for HTTP 405 or + `[("WWW-Authenticate", "Digest")]` for HTTP 401. May be empty. + """ + + http_code = 500 # default; subclasses should override + + def __init__(self, details): + super().__init__(details) + self.headers = [] + + +class InvalidArgumentError(PublicError): + """Client specified an invalid argument. + + The text of this error is assumed not to contain sensitive data, + and so may appear in (e.g.) the response body of a failed HTTP + request. + + Corresponds to HTTP 400 Bad Request or Google error code `INVALID_ARGUMENT`. + """ + + http_code = 400 + + def __init__(self, details=None): + msg = _format_message("Invalid argument", details) + super().__init__(msg) + + +class NotFoundError(PublicError): + """Some requested entity (e.g., file or directory) was not found. + + The text of this error is assumed not to contain sensitive data, + and so may appear in (e.g.) the response body of a failed HTTP + request. + + Corresponds to HTTP 404 Not Found or Google error code `NOT_FOUND`. + """ + + http_code = 404 + + def __init__(self, details=None): + msg = _format_message("Not found", details) + super().__init__(msg) + + +class UnauthenticatedError(PublicError): + """Request does not have valid authentication credentials for the operation. + + The text of this error is assumed not to contain sensitive data, + and so may appear in (e.g.) the response body of a failed HTTP + request. + + Corresponds to HTTP 401 Unauthorized (despite the name) or Google + error code `UNAUTHENTICATED`. HTTP 401 responses are required to + contain a `WWW-Authenticate` challenge, so `UnauthenticatedError` + values are, too. + """ + + http_code = 401 + + def __init__(self, details=None, *, challenge): + """Initialize an `UnauthenticatedError`. + + Args; + details: Optional public, user-facing error message, as a + string or any value that can be converted to string, or + `None` to omit details. + challenge: String value of the `WWW-Authenticate` HTTP header + as described in RFC 7235. + """ + msg = _format_message("Unauthenticated", details) + super().__init__(msg) + self.headers.append(("WWW-Authenticate", challenge)) + + +class PermissionDeniedError(PublicError): + """The caller does not have permission to execute the specified operation. + + The text of this error is assumed not to contain sensitive data, + and so may appear in (e.g.) the response body of a failed HTTP + request. + + Corresponds to HTTP 403 Forbidden or Google error code `PERMISSION_DENIED`. + """ + + http_code = 403 + + def __init__(self, details=None): + msg = _format_message("Permission denied", details) + super().__init__(msg) + + +def _format_message(code_name, details): + if details is None: + return code_name + else: + return "%s: %s" % (code_name, details) diff --git a/lib/python3.12/site-packages/tensorboard/lazy.py b/lib/python3.12/site-packages/tensorboard/lazy.py new file mode 100644 index 0000000000000000000000000000000000000000..686657ed11c444098b7f8990366c416e32af748f --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/lazy.py @@ -0,0 +1,100 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorBoard is a webapp for understanding TensorFlow runs and graphs.""" + + +import functools +import threading +import types + + +def lazy_load(name): + """Decorator to define a function that lazily loads the module 'name'. + + This can be used to defer importing troublesome dependencies - e.g. ones that + are large and infrequently used, or that cause a dependency cycle - + until they are actually used. + + Args: + name: the fully-qualified name of the module; typically the last segment + of 'name' matches the name of the decorated function + + Returns: + Decorator function that produces a lazy-loading module 'name' backed by the + underlying decorated function. + """ + + def wrapper(load_fn): + # Wrap load_fn to call it exactly once and update __dict__ afterwards to + # make future lookups efficient (only failed lookups call __getattr__). + @_memoize + def load_once(self): + if load_once.loading: + raise ImportError( + "Circular import when resolving LazyModule %r" % name + ) + load_once.loading = True + try: + module = load_fn() + finally: + load_once.loading = False + self.__dict__.update(module.__dict__) + load_once.loaded = True + return module + + load_once.loading = False + load_once.loaded = False + + # Define a module that proxies getattr() and dir() to the result of calling + # load_once() the first time it's needed. The class is nested so we can close + # over load_once() and avoid polluting the module's attrs with our own state. + class LazyModule(types.ModuleType): + def __getattr__(self, attr_name): + return getattr(load_once(self), attr_name) + + def __dir__(self): + return dir(load_once(self)) + + def __repr__(self): + if load_once.loaded: + return "<%r via LazyModule (loaded)>" % load_once(self) + return ( + "" + % self.__name__ + ) + + return LazyModule(name) + + return wrapper + + +def _memoize(f): + """Memoizing decorator for f, which must have exactly 1 hashable + argument.""" + nothing = object() # Unique "no value" sentinel object. + cache = {} + # Use a reentrant lock so that if f references the resulting wrapper we die + # with recursion depth exceeded instead of deadlocking. + lock = threading.RLock() + + @functools.wraps(f) + def wrapper(arg): + if cache.get(arg, nothing) is nothing: + with lock: + if cache.get(arg, nothing) is nothing: + cache[arg] = f(arg) + return cache[arg] + + return wrapper diff --git a/lib/python3.12/site-packages/tensorboard/main.py b/lib/python3.12/site-packages/tensorboard/main.py new file mode 100644 index 0000000000000000000000000000000000000000..6577c90d000ac1d88e91d60f411c4a1de2048b85 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/main.py @@ -0,0 +1,48 @@ +# Copyright 2015 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorBoard main module. + +This module ties together `tensorboard.program` and +`tensorboard.default_plugins` to provide standard TensorBoard. It's +meant to be tiny and act as little other than a config file. Those +wishing to customize the set of plugins or static assets that +TensorBoard uses can swap out this file with their own. +""" + +import sys + +from absl import app +from tensorboard import default +from tensorboard import main_lib +from tensorboard import program +from tensorboard.plugins import base_plugin +from tensorboard.util import tb_logging + +logger = tb_logging.get_logger() + + +def run_main(): + """Initializes flags and calls main().""" + main_lib.global_init() + tensorboard = program.TensorBoard(plugins=default.get_plugins()) + try: + app.run(tensorboard.main, flags_parser=tensorboard.configure) + except base_plugin.FlagsError as e: + print("Error: %s" % e, file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + run_main() diff --git a/lib/python3.12/site-packages/tensorboard/main_lib.py b/lib/python3.12/site-packages/tensorboard/main_lib.py new file mode 100644 index 0000000000000000000000000000000000000000..41ef98b1425e022f52be3994ad8c9fe04915a477 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/main_lib.py @@ -0,0 +1,63 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Helpers for TensorBoard main module.""" + +import os +import sys + +import absl.logging + +from tensorboard.compat import tf + + +_TBDEV_SHUTDOWN_MESSAGE = """\ +====================================================================== +ERROR: The `tensorboard dev` command is no longer available. + +TensorBoard.dev has been shut down. For further information, +see the FAQ at . +====================================================================== +""" + + +def global_init(): + """Modifies the global environment for running TensorBoard as main. + + This functions changes global state in the Python process, so it should + not be called from library routines. + """ + # TF versions prior to 1.15.0 included default GCS filesystem caching logic + # that interacted pathologically with the pattern of reads used by TensorBoard + # for logdirs. See: https://github.com/tensorflow/tensorboard/issues/1225 + # The problematic behavior was fixed in 1.15.0 by + # https://github.com/tensorflow/tensorflow/commit/e43b94649d3e1ac5d538e4eca9166b899511d681 + # but for older versions of TF, we avoid a regression by setting this env var to + # disable the cache, which must be done before the first import of tensorflow. + os.environ["GCS_READ_CACHE_DISABLED"] = "1" + + if getattr(tf, "__version__", "stub") == "stub": + print( + "TensorFlow installation not found - running with reduced feature set.", + file=sys.stderr, + ) + + # Only emit log messages at WARNING and above by default to reduce spam. + absl.logging.set_verbosity(absl.logging.WARNING) + + # Intercept attempts to invoke `tensorboard dev` and print turndown message. + if sys.argv[1:] and sys.argv[1] == "dev": + sys.stderr.write(_TBDEV_SHUTDOWN_MESSAGE) + sys.stderr.flush() + sys.exit(1) diff --git a/lib/python3.12/site-packages/tensorboard/manager.py b/lib/python3.12/site-packages/tensorboard/manager.py new file mode 100644 index 0000000000000000000000000000000000000000..823735bd47784db00013e7216b6c486620d229be --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/manager.py @@ -0,0 +1,488 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Private utilities for managing multiple TensorBoard processes.""" + + +import base64 +import dataclasses +import datetime +import errno +import json +import os +import subprocess +import tempfile +import time +import typing + +from typing import Optional + +from tensorboard import version +from tensorboard.util import tb_logging + + +@dataclasses.dataclass(frozen=True) +class TensorBoardInfo: + """Holds the information about a running TensorBoard instance. + + Attributes: + version: Version of the running TensorBoard. + start_time: Seconds since epoch. + pid: ID of the process running TensorBoard. + port: Port on which TensorBoard is running. + path_prefix: Relative prefix to the path, may be empty. + logdir: Data location used by the TensorBoard server, may be empty. + db: Database connection used by the TensorBoard server, may be empty. + cache_key: Opaque, as given by `cache_key` below. + """ + + version: str + start_time: int + pid: int + port: int + path_prefix: str + logdir: str + db: str + cache_key: str + + +def data_source_from_info(info): + """Format the data location for the given TensorBoardInfo. + + Args: + info: A TensorBoardInfo value. + + Returns: + A human-readable string describing the logdir or database connection + used by the server: e.g., "logdir /tmp/logs". + """ + if info.db: + return "db %s" % info.db + else: + return "logdir %s" % info.logdir + + +def _info_to_string(info): + """Convert a `TensorBoardInfo` to string form to be stored on disk. + + The format returned by this function is opaque and should only be + interpreted by `_info_from_string`. + + Args: + info: A valid `TensorBoardInfo` object. + + Raises: + ValueError: If any field on `info` is not of the correct type. + + Returns: + A string representation of the provided `TensorBoardInfo`. + """ + field_name_to_type = typing.get_type_hints(TensorBoardInfo) + for key, field_type in field_name_to_type.items(): + if not isinstance(getattr(info, key), field_type): + raise ValueError( + "expected %r of type %s, but found: %r" + % (key, field_type, getattr(info, key)) + ) + if info.version != version.VERSION: + raise ValueError( + "expected 'version' to be %r, but found: %r" + % (version.VERSION, info.version) + ) + json_value = dataclasses.asdict(info) + return json.dumps(json_value, sort_keys=True, indent=4) + + +def _info_from_string(info_string): + """Parse a `TensorBoardInfo` object from its string representation. + + Args: + info_string: A string representation of a `TensorBoardInfo`, as + produced by a previous call to `_info_to_string`. + + Returns: + A `TensorBoardInfo` value. + + Raises: + ValueError: If the provided string is not valid JSON, or if it is + missing any required fields, or if any field is of incorrect type. + """ + field_name_to_type = typing.get_type_hints(TensorBoardInfo) + try: + json_value = json.loads(info_string) + except ValueError: + raise ValueError("invalid JSON: %r" % (info_string,)) + if not isinstance(json_value, dict): + raise ValueError("not a JSON object: %r" % (json_value,)) + expected_keys = frozenset(field_name_to_type.keys()) + actual_keys = frozenset(json_value) + missing_keys = expected_keys - actual_keys + if missing_keys: + raise ValueError( + "TensorBoardInfo missing keys: %r" % (sorted(missing_keys),) + ) + # For forward compatibility, silently ignore unknown keys. + + # Validate and deserialize fields. + fields = {} + for key, field_type in field_name_to_type.items(): + if not isinstance(json_value[key], field_type): + raise ValueError( + "expected %r of type %s, but found: %r" + % (key, field_type, json_value[key]) + ) + fields[key] = json_value[key] + + return TensorBoardInfo(**fields) + + +def cache_key(working_directory, arguments, configure_kwargs): + """Compute a `TensorBoardInfo.cache_key` field. + + The format returned by this function is opaque. Clients may only + inspect it by comparing it for equality with other results from this + function. + + Args: + working_directory: The directory from which TensorBoard was launched + and relative to which paths like `--logdir` and `--db` are + resolved. + arguments: The command-line args to TensorBoard, as `sys.argv[1:]`. + Should be a list (or tuple), not an unparsed string. If you have a + raw shell command, use `shlex.split` before passing it to this + function. + configure_kwargs: A dictionary of additional argument values to + override the textual `arguments`, with the same semantics as in + `tensorboard.program.TensorBoard.configure`. May be an empty + dictionary. + + Returns: + A string such that if two (prospective or actual) TensorBoard + invocations have the same cache key then it is safe to use one in + place of the other. The converse is not guaranteed: it is often safe + to change the order of TensorBoard arguments, or to explicitly set + them to their default values, or to move them between `arguments` + and `configure_kwargs`, but such invocations may yield distinct + cache keys. + """ + if not isinstance(arguments, (list, tuple)): + raise TypeError( + "'arguments' should be a list of arguments, but found: %r " + "(use `shlex.split` if given a string)" % (arguments,) + ) + datum = { + "working_directory": working_directory, + "arguments": arguments, + "configure_kwargs": configure_kwargs, + } + raw = base64.b64encode( + json.dumps(datum, sort_keys=True, separators=(",", ":")).encode("utf-8") + ) + # `raw` is of type `bytes`, even though it only contains ASCII + # characters; we want it to be `str` in both Python 2 and 3. + return str(raw.decode("ascii")) + + +def _get_info_dir(): + """Get path to directory in which to store info files. + + The directory returned by this function is "owned" by this module. If + the contents of the directory are modified other than via the public + functions of this module, subsequent behavior is undefined. + + The directory will be created if it does not exist. + """ + path = os.path.join(tempfile.gettempdir(), ".tensorboard-info") + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise + else: + os.chmod(path, 0o777) + return path + + +def _get_info_file_path(): + """Get path to info file for the current process. + + As with `_get_info_dir`, the info directory will be created if it + does not exist. + """ + return os.path.join(_get_info_dir(), "pid-%d.info" % os.getpid()) + + +def write_info_file(tensorboard_info): + """Write TensorBoardInfo to the current process's info file. + + This should be called by `main` once the server is ready. When the + server shuts down, `remove_info_file` should be called. + + Args: + tensorboard_info: A valid `TensorBoardInfo` object. + + Raises: + ValueError: If any field on `info` is not of the correct type. + """ + payload = "%s\n" % _info_to_string(tensorboard_info) + with open(_get_info_file_path(), "w") as outfile: + outfile.write(payload) + + +def remove_info_file(): + """Remove the current process's TensorBoardInfo file, if it exists. + + If the file does not exist, no action is taken and no error is + raised. + """ + try: + os.unlink(_get_info_file_path()) + except OSError as e: + if e.errno == errno.ENOENT: + # The user may have wiped their temporary directory or something. + # Not a problem: we're already in the state that we want to be in. + pass + else: + raise + + +def get_all(): + """Return TensorBoardInfo values for running TensorBoard processes. + + This function may not provide a perfect snapshot of the set of running + processes. Its result set may be incomplete if the user has cleaned + their /tmp/ directory while TensorBoard processes are running. It may + contain extraneous entries if TensorBoard processes exited uncleanly + (e.g., with SIGKILL or SIGQUIT). + + Entries in the info directory that do not represent valid + `TensorBoardInfo` values will be silently ignored. + + Returns: + A fresh list of `TensorBoardInfo` objects. + """ + info_dir = _get_info_dir() + results = [] + for filename in os.listdir(info_dir): + filepath = os.path.join(info_dir, filename) + try: + with open(filepath) as infile: + contents = infile.read() + except IOError as e: + if e.errno == errno.EACCES: + # May have been written by this module in a process whose + # `umask` includes some bits of 0o444. + continue + else: + raise + try: + info = _info_from_string(contents) + except ValueError: + # Ignore unrecognized files, logging at debug only. + tb_logging.get_logger().debug( + "invalid info file: %r", + filepath, + exc_info=True, + ) + else: + results.append(info) + return results + + +@dataclasses.dataclass(frozen=True) +class StartReused: + """Possible return value of the `start` function. + + Indicates that a call to `start` was compatible with an existing + TensorBoard process, which can be reused according to the provided + info. + + Attributes: + info: A `TensorBoardInfo` object. + """ + + info: TensorBoardInfo + + +@dataclasses.dataclass(frozen=True) +class StartLaunched: + """Possible return value of the `start` function. + + Indicates that a call to `start` successfully launched a new + TensorBoard process, which is available with the provided info. + + Attributes: + info: A `TensorBoardInfo` object. + """ + + info: TensorBoardInfo + + +@dataclasses.dataclass(frozen=True) +class StartFailed: + """Possible return value of the `start` function. + + Indicates that a call to `start` tried to launch a new TensorBoard + instance, but the subprocess exited with the given exit code and + output streams. (If the contents of the output streams are no longer + available---e.g., because the user has emptied /tmp/---then the + corresponding values will be `None`.) + + Attributes: + exit_code: As `Popen.returncode` (negative for signal). + stdout: Error message to stdout if the stream could not be read. + stderr: Error message to stderr if the stream could not be read. + """ + + exit_code: int + stdout: Optional[str] + stderr: Optional[str] + + +@dataclasses.dataclass(frozen=True) +class StartExecFailed: + """Possible return value of the `start` function. + + Indicates that a call to `start` failed to invoke the subprocess. + + Attributes: + os_error: `OSError` due to `Popen` invocation. + explicit_binary: If the TensorBoard executable was chosen via the + `TENSORBOARD_BINARY` environment variable, then this field contains + the path to that binary; otherwise `None`. + """ + + os_error: OSError + explicit_binary: Optional[str] + + +@dataclasses.dataclass(frozen=True) +class StartTimedOut: + """Possible return value of the `start` function. + + Indicates that a call to `start` launched a TensorBoard process, but + that process neither exited nor wrote its info file within the allowed + timeout period. The process may still be running under the included + PID. + + Attributes: + pid: ID of the process running TensorBoard. + """ + + pid: int + + +def start(arguments, timeout=datetime.timedelta(seconds=60)): + """Start a new TensorBoard instance, or reuse a compatible one. + + If the cache key determined by the provided arguments and the current + working directory (see `cache_key`) matches the cache key of a running + TensorBoard process (see `get_all`), that process will be reused. + + Otherwise, a new TensorBoard process will be spawned with the provided + arguments, using the `tensorboard` binary from the system path. + + Args: + arguments: List of strings to be passed as arguments to + `tensorboard`. (If you have a raw command-line string, see + `shlex.split`.) + timeout: `datetime.timedelta` object describing how long to wait for + the subprocess to initialize a TensorBoard server and write its + `TensorBoardInfo` file. If the info file is not written within + this time period, `start` will assume that the subprocess is stuck + in a bad state, and will give up on waiting for it and return a + `StartTimedOut` result. Note that in such a case the subprocess + will not be killed. Default value is 60 seconds. + + Returns: + A `StartReused`, `StartLaunched`, `StartFailed`, or `StartTimedOut` + object. + """ + this_cache_key = cache_key( + working_directory=os.getcwd(), + arguments=arguments, + configure_kwargs={}, + ) + match = _find_matching_instance(this_cache_key) + if match: + return StartReused(info=match) + + (stdout_fd, stdout_path) = tempfile.mkstemp(prefix=".tensorboard-stdout-") + (stderr_fd, stderr_path) = tempfile.mkstemp(prefix=".tensorboard-stderr-") + start_time_seconds = time.time() + explicit_tb = os.environ.get("TENSORBOARD_BINARY", None) + try: + p = subprocess.Popen( + ["tensorboard" if explicit_tb is None else explicit_tb] + arguments, + stdout=stdout_fd, + stderr=stderr_fd, + ) + except OSError as e: + return StartExecFailed(os_error=e, explicit_binary=explicit_tb) + finally: + os.close(stdout_fd) + os.close(stderr_fd) + + poll_interval_seconds = 0.5 + end_time_seconds = start_time_seconds + timeout.total_seconds() + while time.time() < end_time_seconds: + time.sleep(poll_interval_seconds) + subprocess_result = p.poll() + if subprocess_result is not None: + return StartFailed( + exit_code=subprocess_result, + stdout=_maybe_read_file(stdout_path), + stderr=_maybe_read_file(stderr_path), + ) + info = _find_matching_instance(this_cache_key) + if info: + # Don't check that `info.pid == p.pid`, since on Windows that may + # not be the case: see #4300. + return StartLaunched(info=info) + else: + return StartTimedOut(pid=p.pid) + + +def _find_matching_instance(cache_key): + """Find a running TensorBoard instance compatible with the cache key. + + Returns: + A `TensorBoardInfo` object, or `None` if none matches the cache key. + """ + infos = get_all() + candidates = [info for info in infos if info.cache_key == cache_key] + for candidate in sorted(candidates, key=lambda x: x.port): + # TODO(@wchargin): Check here that the provided port is still live. + return candidate + return None + + +def _maybe_read_file(filename): + """Read the given file, if it exists. + + Args: + filename: A path to a file. + + Returns: + A string containing the file contents, or `None` if the file does + not exist. + """ + try: + with open(filename) as infile: + return infile.read() + except IOError as e: + if e.errno == errno.ENOENT: + return None diff --git a/lib/python3.12/site-packages/tensorboard/notebook.py b/lib/python3.12/site-packages/tensorboard/notebook.py new file mode 100644 index 0000000000000000000000000000000000000000..bcfd6352778256f3aa1f5c4b6742d99b3e272075 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/notebook.py @@ -0,0 +1,443 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for using TensorBoard in notebook contexts, like Colab. + +These APIs are experimental and subject to change. +""" + + +import datetime +import errno +import html +import json +import os +import random +import shlex +import textwrap +import time + +from tensorboard import manager + + +# Return values for `_get_context` (see that function's docs for +# details). +_CONTEXT_COLAB = "_CONTEXT_COLAB" +_CONTEXT_IPYTHON = "_CONTEXT_IPYTHON" +_CONTEXT_NONE = "_CONTEXT_NONE" + + +def _get_context(): + """Determine the most specific context that we're in. + + Returns: + _CONTEXT_COLAB: If in Colab with an IPython notebook context. + _CONTEXT_IPYTHON: If not in Colab, but we are in an IPython notebook + context (e.g., from running `jupyter notebook` at the command + line). + _CONTEXT_NONE: Otherwise (e.g., by running a Python script at the + command-line or using the `ipython` interactive shell). + """ + # In Colab, the `google.colab` module is available, but the shell + # returned by `IPython.get_ipython` does not have a `get_trait` + # method. + try: + import google.colab # noqa: F401 + import IPython + except ImportError: + pass + else: + if IPython.get_ipython() is not None: + # We'll assume that we're in a Colab notebook context. + return _CONTEXT_COLAB + + # In an IPython command line shell or Jupyter notebook, we can + # directly query whether we're in a notebook context. + try: + import IPython + except ImportError: + pass + else: + ipython = IPython.get_ipython() + if ipython is not None and ipython.has_trait("kernel"): + return _CONTEXT_IPYTHON + + # Otherwise, we're not in a known notebook context. + return _CONTEXT_NONE + + +def load_ipython_extension(ipython): + """Deprecated: use `%load_ext tensorboard` instead. + + Raises: + RuntimeError: Always. + """ + raise RuntimeError( + "Use '%load_ext tensorboard' instead of '%load_ext tensorboard.notebook'." + ) + + +def _load_ipython_extension(ipython): + """Load the TensorBoard notebook extension. + + Intended to be called from `%load_ext tensorboard`. Do not invoke this + directly. + + Args: + ipython: An `IPython.InteractiveShell` instance. + """ + _register_magics(ipython) + + +def _register_magics(ipython): + """Register IPython line/cell magics. + + Args: + ipython: An `InteractiveShell` instance. + """ + ipython.register_magic_function( + _start_magic, + magic_kind="line", + magic_name="tensorboard", + ) + + +def _start_magic(line): + """Implementation of the `%tensorboard` line magic.""" + return start(line) + + +def start(args_string): + """Launch and display a TensorBoard instance as if at the command line. + + Args: + args_string: Command-line arguments to TensorBoard, to be + interpreted by `shlex.split`: e.g., "--logdir ./logs --port 0". + Shell metacharacters are not supported: e.g., "--logdir 2>&1" will + point the logdir at the literal directory named "2>&1". + """ + context = _get_context() + try: + import IPython + import IPython.display + except ImportError: + IPython = None + + if context == _CONTEXT_NONE: + handle = None + print("Launching TensorBoard...") + else: + handle = IPython.display.display( + IPython.display.Pretty("Launching TensorBoard..."), + display_id=True, + ) + + def print_or_update(message): + if handle is None: + print(message) + else: + handle.update(IPython.display.Pretty(message)) + + parsed_args = shlex.split(args_string, comments=True, posix=True) + start_result = manager.start(parsed_args) + + if isinstance(start_result, manager.StartLaunched): + _display( + port=start_result.info.port, + print_message=False, + display_handle=handle, + ) + + elif isinstance(start_result, manager.StartReused): + template = ( + "Reusing TensorBoard on port {port} (pid {pid}), started {delta} ago. " + "(Use '!kill {pid}' to kill it.)" + ) + message = template.format( + port=start_result.info.port, + pid=start_result.info.pid, + delta=_time_delta_from_info(start_result.info), + ) + print_or_update(message) + _display( + port=start_result.info.port, + print_message=False, + display_handle=None, + ) + + elif isinstance(start_result, manager.StartFailed): + + def format_stream(name, value): + if value == "": + return "" + elif value is None: + return "\n" % name + else: + return "\nContents of %s:\n%s" % (name, value.strip()) + + message = ( + "ERROR: Failed to launch TensorBoard (exited with %d).%s%s" + % ( + start_result.exit_code, + format_stream("stderr", start_result.stderr), + format_stream("stdout", start_result.stdout), + ) + ) + print_or_update(message) + + elif isinstance(start_result, manager.StartExecFailed): + the_tensorboard_binary = ( + "%r (set by the `TENSORBOARD_BINARY` environment variable)" + % (start_result.explicit_binary,) + if start_result.explicit_binary is not None + else "`tensorboard`" + ) + if start_result.os_error.errno == errno.ENOENT: + message = ( + "ERROR: Could not find %s. Please ensure that your PATH contains " + "an executable `tensorboard` program, or explicitly specify the path " + "to a TensorBoard binary by setting the `TENSORBOARD_BINARY` " + "environment variable." % (the_tensorboard_binary,) + ) + else: + message = "ERROR: Failed to start %s: %s" % ( + the_tensorboard_binary, + start_result.os_error, + ) + print_or_update(textwrap.fill(message)) + + elif isinstance(start_result, manager.StartTimedOut): + message = ( + "ERROR: Timed out waiting for TensorBoard to start. " + "It may still be running as pid %d." % start_result.pid + ) + print_or_update(message) + + else: + raise TypeError( + "Unexpected result from `manager.start`: %r.\n" + "This is a TensorBoard bug; please report it." % start_result + ) + + +def _time_delta_from_info(info): + """Format the elapsed time for the given TensorBoardInfo. + + Args: + info: A TensorBoardInfo value. + + Returns: + A human-readable string describing the time since the server + described by `info` started: e.g., "2 days, 0:48:58". + """ + delta_seconds = int(time.time()) - info.start_time + return str(datetime.timedelta(seconds=delta_seconds)) + + +def display(port=None, height=None): + """Display a TensorBoard instance already running on this machine. + + Args: + port: The port on which the TensorBoard server is listening, as an + `int`, or `None` to automatically select the most recently + launched TensorBoard. + height: The height of the frame into which to render the TensorBoard + UI, as an `int` number of pixels, or `None` to use a default value + (currently 800). + """ + _display(port=port, height=height, print_message=True, display_handle=None) + + +def _display(port=None, height=None, print_message=False, display_handle=None): + """Internal version of `display`. + + Args: + port: As with `display`. + height: As with `display`. + print_message: True to print which TensorBoard instance was selected + for display (if applicable), or False otherwise. + display_handle: If not None, an IPython display handle into which to + render TensorBoard. + """ + if height is None: + height = 800 + + if port is None: + infos = manager.get_all() + if not infos: + raise ValueError( + "Can't display TensorBoard: no known instances running." + ) + else: + info = max(manager.get_all(), key=lambda x: x.start_time) + port = info.port + else: + infos = [i for i in manager.get_all() if i.port == port] + info = max(infos, key=lambda x: x.start_time) if infos else None + + if print_message: + if info is not None: + message = ( + "Selecting TensorBoard with {data_source} " + "(started {delta} ago; port {port}, pid {pid})." + ).format( + data_source=manager.data_source_from_info(info), + delta=_time_delta_from_info(info), + port=info.port, + pid=info.pid, + ) + print(message) + else: + # The user explicitly provided a port, and we don't have any + # additional information. There's nothing useful to say. + pass + + fn = { + _CONTEXT_COLAB: _display_colab, + _CONTEXT_IPYTHON: _display_ipython, + _CONTEXT_NONE: _display_cli, + }[_get_context()] + return fn(port=port, height=height, display_handle=display_handle) + + +def _display_colab(port, height, display_handle): + """Display a TensorBoard instance in a Colab output frame. + + The Colab VM is not directly exposed to the network, so the Colab + runtime provides a service worker tunnel to proxy requests from the + end user's browser through to servers running on the Colab VM: the + output frame may issue requests to https://localhost: (HTTPS + only), which will be forwarded to the specified port on the VM. + + It does not suffice to create an `iframe` and let the service worker + redirect its traffic (` + + """ + proxy_url = os.environ.get("TENSORBOARD_PROXY_URL") + if proxy_url is not None: + # Allow %PORT% in $TENSORBOARD_PROXY_URL + proxy_url = proxy_url.replace("%PORT%", "%d" % port) + replacements = [ + ("%HTML_ID%", html.escape(frame_id, quote=True)), + ("%JSON_ID%", json.dumps(frame_id)), + ("%HEIGHT%", "%d" % height), + ("%PORT%", "0"), + ("%URL%", json.dumps(proxy_url)), + ] + else: + replacements = [ + ("%HTML_ID%", html.escape(frame_id, quote=True)), + ("%JSON_ID%", json.dumps(frame_id)), + ("%HEIGHT%", "%d" % height), + ("%PORT%", "%d" % port), + ("%URL%", json.dumps("/")), + ] + + for k, v in replacements: + shell = shell.replace(k, v) + iframe = IPython.display.HTML(shell) + if display_handle: + display_handle.update(iframe) + else: + IPython.display.display(iframe) + + +def _display_cli(port, height, display_handle): + del height # unused + del display_handle # unused + message = "Please visit http://localhost:%d in a web browser." % port + print(message) + + +def list(): + """Print a listing of known running TensorBoard instances. + + TensorBoard instances that were killed uncleanly (e.g., with SIGKILL + or SIGQUIT) may appear in this list even if they are no longer + running. Conversely, this list may be missing some entries if your + operating system's temporary directory has been cleared since a + still-running TensorBoard instance started. + """ + infos = manager.get_all() + if not infos: + print("No known TensorBoard instances running.") + return + + print("Known TensorBoard instances:") + for info in infos: + template = ( + " - port {port}: {data_source} (started {delta} ago; pid {pid})" + ) + print( + template.format( + port=info.port, + data_source=manager.data_source_from_info(info), + delta=_time_delta_from_info(info), + pid=info.pid, + ) + ) diff --git a/lib/python3.12/site-packages/tensorboard/plugin_util.py b/lib/python3.12/site-packages/tensorboard/plugin_util.py new file mode 100644 index 0000000000000000000000000000000000000000..82de22f9412a763c23127f6282b63b06baa8eb03 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugin_util.py @@ -0,0 +1,267 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides utilities that may be especially useful to plugins.""" + +from google.protobuf import json_format +from importlib import metadata +from packaging import version +import threading + +from tensorboard._vendor.bleach.sanitizer import Cleaner +import markdown + +from tensorboard import context as _context +from tensorboard.backend import experiment_id as _experiment_id +from tensorboard.util import tb_logging + + +logger = tb_logging.get_logger() + +_ALLOWED_ATTRIBUTES = { + "a": ["href", "title"], + "img": ["src", "title", "alt"], +} + +_ALLOWED_TAGS = [ + "ul", + "ol", + "li", + "p", + "pre", + "code", + "blockquote", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "hr", + "br", + "strong", + "em", + "a", + "img", + "table", + "thead", + "tbody", + "td", + "tr", + "th", +] + + +# Cache Markdown converter to avoid expensive initialization at each +# call to `markdown_to_safe_html`. Cache a different instance per thread. +class _MarkdownStore(threading.local): + def __init__(self): + self.markdown = markdown.Markdown( + extensions=[ + "markdown.extensions.tables", + "markdown.extensions.fenced_code", + ] + ) + + +_MARKDOWN_STORE = _MarkdownStore() + + +# Cache Cleaner to avoid expensive initialization at each call to `clean`. +# Cache a different instance per thread. +class _CleanerStore(threading.local): + def __init__(self): + self.cleaner = Cleaner( + tags=_ALLOWED_TAGS, attributes=_ALLOWED_ATTRIBUTES + ) + + +_CLEANER_STORE = _CleanerStore() + + +def safe_html(unsafe_string): + """Return the input as a str, sanitized for insertion into the DOM. + + Arguments: + unsafe_string: A Unicode string or UTF-8--encoded bytestring + possibly containing unsafe HTML markup. + + Returns: + A string containing safe HTML. + """ + total_null_bytes = 0 + if isinstance(unsafe_string, bytes): + unsafe_string = unsafe_string.decode("utf-8") + return _CLEANER_STORE.cleaner.clean(unsafe_string) + + +def markdown_to_safe_html(markdown_string): + """Convert Markdown to HTML that's safe to splice into the DOM. + + Arguments: + markdown_string: A Unicode string or UTF-8--encoded bytestring + containing Markdown source. Markdown tables are supported. + + Returns: + A string containing safe HTML. + """ + return markdowns_to_safe_html([markdown_string], lambda xs: xs[0]) + + +def markdowns_to_safe_html(markdown_strings, combine): + """Convert multiple Markdown documents to one safe HTML document. + + One could also achieve this by calling `markdown_to_safe_html` + multiple times and combining the results. Compared to that approach, + this function may be faster, because HTML sanitization (which can be + expensive) is performed only once rather than once per input. It may + also be less precise: if one of the input documents has unsafe HTML + that is sanitized away, that sanitization might affect other + documents, even if those documents are safe. + + Args: + markdown_strings: List of Markdown source strings to convert, as + Unicode strings or UTF-8--encoded bytestrings. Markdown tables + are supported. + combine: Callback function that takes a list of unsafe HTML + strings of the same shape as `markdown_strings` and combines + them into a single unsafe HTML string, which will be sanitized + and returned. + + Returns: + A string containing safe HTML. + """ + unsafe_htmls = [] + total_null_bytes = 0 + + for source in markdown_strings: + # Convert to utf-8 whenever we have a binary input. + if isinstance(source, bytes): + source_decoded = source.decode("utf-8") + # Remove null bytes and warn if there were any, since it probably means + # we were given a bad encoding. + source = source_decoded.replace("\x00", "") + total_null_bytes += len(source_decoded) - len(source) + unsafe_html = _MARKDOWN_STORE.markdown.convert(source) + unsafe_htmls.append(unsafe_html) + + unsafe_combined = combine(unsafe_htmls) + sanitized_combined = _CLEANER_STORE.cleaner.clean(unsafe_combined) + + warning = "" + if total_null_bytes: + warning = ( + "\n" + ) % total_null_bytes + + return warning + sanitized_combined + + +def context(environ): + """Get a TensorBoard `RequestContext` from a WSGI environment. + + Returns: + A `RequestContext` value. + """ + return _context.from_environ(environ) + + +def experiment_id(environ): + """Determine the experiment ID associated with a WSGI request. + + Each request to TensorBoard has an associated experiment ID, which is + always a string and may be empty. This experiment ID should be passed + to data providers. + + Args: + environ: A WSGI environment `dict`. For a Werkzeug request, this is + `request.environ`. + + Returns: + A experiment ID, as a possibly-empty `str`. + """ + return environ.get(_experiment_id.WSGI_ENVIRON_KEY, "") + + +def proto_to_json(proto): + """Utility method to convert proto to JSON, accounting for different version support. + + Args: + proto: The proto to convert to JSON. + """ + # Fallback for internal usage, since non third-party code doesn't really + # have the concept of "versions" in a monorepo. The package version chosen + # below is the minimum value to choose the non-deprecated kwarg to + # `MessageToJson`. + try: + current_version = metadata.version("protobuf") + except metadata.PackageNotFoundError: + current_version = "5.0.0" + if version.parse(current_version) >= version.parse("5.0.0"): + return json_format.MessageToJson( + proto, + always_print_fields_with_no_presence=True, + ) + return json_format.MessageToJson( + proto, + including_default_value_fields=True, + ) + + +class _MetadataVersionChecker: + """TensorBoard-internal utility for warning when data is too new. + + Specify a maximum known `version` number as stored in summary + metadata, and automatically reject and warn on data from newer + versions. This keeps a (single) bit of internal state to handle + logging a warning to the user at most once. + + This should only be used by plugins bundled with TensorBoard, since + it may instruct users to upgrade their copy of TensorBoard. + """ + + def __init__(self, data_kind, latest_known_version): + """Initialize a `_MetadataVersionChecker`. + + Args: + data_kind: A human-readable description of the kind of data + being read, like "scalar" or "histogram" or "PR curve". + latest_known_version: Highest tolerated value of `version`, + like `0`. + """ + self._data_kind = data_kind + self._latest_known_version = latest_known_version + self._warned = False + + def ok(self, version, run, tag): + """Test whether `version` is permitted, else complain.""" + if 0 <= version <= self._latest_known_version: + return True + self._maybe_warn(version, run, tag) + return False + + def _maybe_warn(self, version, run, tag): + if self._warned: + return + self._warned = True + logger.warning( + "Some %s data is too new to be read by this version of TensorBoard. " + "Upgrading TensorBoard may fix this. " + "(sample: run %r, tag %r, data version %r)", + self._data_kind, + run, + tag, + version, + ) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/__init__.py b/lib/python3.12/site-packages/tensorboard/plugins/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/__init__.py b/lib/python3.12/site-packages/tensorboard/plugins/audio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..73c5545392df000230ec37266a42a6bab61e1f34 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/audio_plugin.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/audio_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..85094edd8054f11befbc005f2d8f399d70f57c52 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/audio_plugin.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/metadata.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4f97d67d42d729ba04d77d109c4d8557becd8eaa Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/metadata.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/plugin_data_pb2.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/plugin_data_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9f1d036d992432d2a9e65d218f75c03f1a955e77 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/plugin_data_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/summary.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/summary.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e6392493de9b964897091acca7c6f833f41f9da Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/summary.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/summary_v2.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/summary_v2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08365e0eeeb24c4f207e6afa2b219b4b465fd837 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/audio/__pycache__/summary_v2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/audio_plugin.py b/lib/python3.12/site-packages/tensorboard/plugins/audio/audio_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..0ed305f702ee1946c422d919c88c734c20257e04 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/audio/audio_plugin.py @@ -0,0 +1,231 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The TensorBoard Audio plugin.""" + + +import urllib.parse + +from werkzeug import wrappers + +from tensorboard import errors +from tensorboard import plugin_util +from tensorboard.backend import http_util +from tensorboard.data import provider +from tensorboard.plugins import base_plugin +from tensorboard.plugins.audio import metadata + + +_DEFAULT_MIME_TYPE = "application/octet-stream" +_DEFAULT_DOWNSAMPLING = 10 # audio clips per time series +_MIME_TYPES = { + metadata.Encoding.Value("WAV"): "audio/wav", +} +_ALLOWED_MIME_TYPES = frozenset( + list(_MIME_TYPES.values()) + [_DEFAULT_MIME_TYPE] +) + + +class AudioPlugin(base_plugin.TBPlugin): + """Audio Plugin for TensorBoard.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates AudioPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._data_provider = context.data_provider + self._downsample_to = (context.sampling_hints or {}).get( + self.plugin_name, _DEFAULT_DOWNSAMPLING + ) + self._version_checker = plugin_util._MetadataVersionChecker( + data_kind="audio", + latest_known_version=0, + ) + + def get_plugin_apps(self): + return { + "/audio": self._serve_audio_metadata, + "/individualAudio": self._serve_individual_audio, + "/tags": self._serve_tags, + } + + def is_active(self): + return False # `list_plugins` as called by TB core suffices + + def frontend_metadata(self): + return base_plugin.FrontendMetadata(element_name="tf-audio-dashboard") + + def _index_impl(self, ctx, experiment): + """Return information about the tags in each run. + + Result is a dictionary of the form + + { + "runName1": { + "tagName1": { + "displayName": "The first tag", + "description": "

Long ago there was just one tag...

", + "samples": 3 + }, + "tagName2": ..., + ... + }, + "runName2": ..., + ... + } + + For each tag, `samples` is the greatest number of audio clips that + appear at any particular step. (It's not related to "samples of a + waveform.") For example, if for tag `minibatch_input` there are + five audio clips at step 0 and ten audio clips at step 1, then the + dictionary for `"minibatch_input"` will contain `"samples": 10`. + """ + mapping = self._data_provider.list_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + ) + result = {run: {} for run in mapping} + for run, tag_to_time_series in mapping.items(): + for tag, time_series in tag_to_time_series.items(): + md = metadata.parse_plugin_metadata(time_series.plugin_content) + if not self._version_checker.ok(md.version, run, tag): + continue + description = plugin_util.markdown_to_safe_html( + time_series.description + ) + result[run][tag] = { + "displayName": time_series.display_name, + "description": description, + "samples": time_series.max_length, + } + return result + + @wrappers.Request.application + def _serve_audio_metadata(self, request): + """Given a tag and list of runs, serve a list of metadata for audio. + + Note that the actual audio data are not sent; instead, we respond + with URLs to the audio. The frontend should treat these URLs as + opaque and should not try to parse information about them or + generate them itself, as the format may change. + + Args: + request: A werkzeug.wrappers.Request object. + + Returns: + A werkzeug.Response application. + """ + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + tag = request.args.get("tag") + run = request.args.get("run") + sample = int(request.args.get("sample", 0)) + + response = self._audio_response_for_run( + ctx, experiment, run, tag, sample + ) + return http_util.Respond(request, response, "application/json") + + def _audio_response_for_run(self, ctx, experiment, run, tag, sample): + """Builds a JSON-serializable object with information about audio. + + Args: + run: The name of the run. + tag: The name of the tag the audio entries all belong to. + sample: The zero-indexed sample of the audio sample for which to + retrieve information. For instance, setting `sample` to `2` will + fetch information about only the third audio clip of each batch, + and steps with fewer than three audio clips will be omitted from + the results. + + Returns: + A list of dictionaries containing the wall time, step, label, + content type, and query string for each audio entry. + """ + all_audio = self._data_provider.read_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + downsample=self._downsample_to, + run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), + ) + audio = all_audio.get(run, {}).get(tag, None) + if audio is None: + raise errors.NotFoundError( + "No audio data for run=%r, tag=%r" % (run, tag) + ) + content_type = self._get_mime_type(ctx, experiment, run, tag) + response = [] + for datum in audio: + if len(datum.values) < sample: + continue + query = urllib.parse.urlencode( + { + "blob_key": datum.values[sample].blob_key, + "content_type": content_type, + } + ) + response.append( + { + "wall_time": datum.wall_time, + "label": "", + "step": datum.step, + "contentType": content_type, + "query": query, + } + ) + return response + + def _get_mime_type(self, ctx, experiment, run, tag): + # TODO(@wchargin): Move this call from `/audio` (called many + # times) to `/tags` (called few times) to reduce data provider + # calls. + mapping = self._data_provider.list_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + ) + time_series = mapping.get(run, {}).get(tag, None) + if time_series is None: + raise errors.NotFoundError( + "No audio data for run=%r, tag=%r" % (run, tag) + ) + parsed = metadata.parse_plugin_metadata(time_series.plugin_content) + return _MIME_TYPES.get(parsed.encoding, _DEFAULT_MIME_TYPE) + + @wrappers.Request.application + def _serve_individual_audio(self, request): + """Serve encoded audio data.""" + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + mime_type = request.args["content_type"] + if mime_type not in _ALLOWED_MIME_TYPES: + raise errors.InvalidArgumentError( + "Illegal mime type %r" % mime_type + ) + blob_key = request.args["blob_key"] + data = self._data_provider.read_blob(ctx, blob_key=blob_key) + return http_util.Respond(request, data, mime_type) + + @wrappers.Request.application + def _serve_tags(self, request): + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + index = self._index_impl(ctx, experiment) + return http_util.Respond(request, index, "application/json") diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/metadata.py b/lib/python3.12/site-packages/tensorboard/plugins/audio/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..5ab35efb162b36b60e99681a5b3b1a92dbda9462 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/audio/metadata.py @@ -0,0 +1,70 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Internal information about the audio plugin.""" + + +from tensorboard.compat.proto import summary_pb2 +from tensorboard.plugins.audio import plugin_data_pb2 + +PLUGIN_NAME = "audio" + +# The most recent value for the `version` field of the `AudioPluginData` +# proto. +PROTO_VERSION = 0 + +# Expose the `Encoding` enum constants. +Encoding = plugin_data_pb2.AudioPluginData.Encoding + + +def create_summary_metadata( + display_name, description, encoding, *, converted_to_tensor=None +): + """Create a `SummaryMetadata` proto for audio plugin data. + + Returns: + A `SummaryMetadata` protobuf object. + """ + content = plugin_data_pb2.AudioPluginData( + version=PROTO_VERSION, + encoding=encoding, + converted_to_tensor=converted_to_tensor, + ) + metadata = summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content.SerializeToString() + ), + ) + return metadata + + +def parse_plugin_metadata(content): + """Parse summary metadata to a Python object. + + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the audio plugin. + + Returns: + An `AudioPluginData` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + result = plugin_data_pb2.AudioPluginData.FromString(content) + if result.version == 0: + return result + # No other versions known at this time, so no migrations to do. + return result diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/plugin_data_pb2.py b/lib/python3.12/site-packages/tensorboard/plugins/audio/plugin_data_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..e51e684adc0690ae86ad81d7b2073a7a5575abb2 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/audio/plugin_data_pb2.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorboard/plugins/audio/plugin_data.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n+tensorboard/plugins/audio/plugin_data.proto\x12\x0btensorboard\"\x9a\x01\n\x0f\x41udioPluginData\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x37\n\x08\x65ncoding\x18\x02 \x01(\x0e\x32%.tensorboard.AudioPluginData.Encoding\x12\x1b\n\x13\x63onverted_to_tensor\x18\x03 \x01(\x08\" \n\x08\x45ncoding\x12\x0b\n\x07UNKNOWN\x10\x00\x12\x07\n\x03WAV\x10\x0b\x62\x06proto3') + + + +_AUDIOPLUGINDATA = DESCRIPTOR.message_types_by_name['AudioPluginData'] +_AUDIOPLUGINDATA_ENCODING = _AUDIOPLUGINDATA.enum_types_by_name['Encoding'] +AudioPluginData = _reflection.GeneratedProtocolMessageType('AudioPluginData', (_message.Message,), { + 'DESCRIPTOR' : _AUDIOPLUGINDATA, + '__module__' : 'tensorboard.plugins.audio.plugin_data_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.AudioPluginData) + }) +_sym_db.RegisterMessage(AudioPluginData) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _AUDIOPLUGINDATA._serialized_start=61 + _AUDIOPLUGINDATA._serialized_end=215 + _AUDIOPLUGINDATA_ENCODING._serialized_start=183 + _AUDIOPLUGINDATA_ENCODING._serialized_end=215 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/summary.py b/lib/python3.12/site-packages/tensorboard/plugins/audio/summary.py new file mode 100644 index 0000000000000000000000000000000000000000..7f089b202e737496097b2a150056d105a1527b15 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/audio/summary.py @@ -0,0 +1,232 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Audio summaries and TensorFlow operations to create them. + +An audio summary stores a rank-2 string tensor of shape `[k, 2]`, where +`k` is the number of audio clips recorded in the summary. Each row of +the tensor is a pair `[encoded_audio, label]`, where `encoded_audio` is +a binary string whose encoding is specified in the summary metadata, and +`label` is a UTF-8 encoded Markdown string describing the audio clip. + +NOTE: This module is in beta, and its API is subject to change, but the +data that it stores to disk will be supported forever. +""" + + +import functools +import warnings + +import numpy as np + +from tensorboard.util import encoder as encoder_util +from tensorboard.plugins.audio import metadata +from tensorboard.plugins.audio import summary_v2 + + +# Export V2 versions. +audio = summary_v2.audio + + +_LABELS_WARNING = ( + "Labels on audio summaries are deprecated and will be removed. " + "See ." +) + + +def op( + name, + audio, + sample_rate, + labels=None, + max_outputs=3, + encoding=None, + display_name=None, + description=None, + collections=None, +): + """Create a legacy audio summary op for use in a TensorFlow graph. + + Arguments: + name: A unique name for the generated summary node. + audio: A `Tensor` representing audio data with shape `[k, t, c]`, + where `k` is the number of audio clips, `t` is the number of + frames, and `c` is the number of channels. Elements should be + floating-point values in `[-1.0, 1.0]`. Any of the dimensions may + be statically unknown (i.e., `None`). + sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the + sample rate, in Hz. Must be positive. + labels: Deprecated. Do not set. + max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this + many audio clips will be emitted at each step. When more than + `max_outputs` many clips are provided, the first `max_outputs` + many clips will be used and the rest silently discarded. + encoding: A constant `str` (not string tensor) indicating the + desired encoding. You can choose any format you like, as long as + it's "wav". Please see the "API compatibility note" below. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A TensorFlow summary op. + + API compatibility note: The default value of the `encoding` + argument is _not_ guaranteed to remain unchanged across TensorBoard + versions. In the future, we will by default encode as FLAC instead of + as WAV. If the specific format is important to you, please provide a + file format explicitly. + """ + if labels is not None: + warnings.warn(_LABELS_WARNING) + + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + if encoding is None: + encoding = "wav" + + if encoding == "wav": + encoding = metadata.Encoding.Value("WAV") + encoder = functools.partial( + tf.audio.encode_wav, sample_rate=sample_rate + ) + else: + raise ValueError("Unknown encoding: %r" % encoding) + + with tf.name_scope(name), tf.control_dependencies( + [tf.assert_rank(audio, 3)] + ): + limited_audio = audio[:max_outputs] + encoded_audio = tf.map_fn( + encoder, limited_audio, dtype=tf.string, name="encode_each_audio" + ) + if labels is None: + limited_labels = tf.tile([""], tf.shape(input=limited_audio)[:1]) + else: + limited_labels = labels[:max_outputs] + tensor = tf.transpose(a=tf.stack([encoded_audio, limited_labels])) + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, + description=description, + encoding=encoding, + ) + return tf.summary.tensor_summary( + name="audio_summary", + tensor=tensor, + collections=collections, + summary_metadata=summary_metadata, + ) + + +def pb( + name, + audio, + sample_rate, + labels=None, + max_outputs=3, + encoding=None, + display_name=None, + description=None, +): + """Create a legacy audio summary protobuf. + + This behaves as if you were to create an `op` with the same arguments + (wrapped with constant tensors where appropriate) and then execute + that summary op in a TensorFlow session. + + Arguments: + name: A unique name for the generated summary node. + audio: An `np.array` representing audio data with shape `[k, t, c]`, + where `k` is the number of audio clips, `t` is the number of + frames, and `c` is the number of channels. Elements should be + floating-point values in `[-1.0, 1.0]`. + sample_rate: An `int` that represents the sample rate, in Hz. + Must be positive. + labels: Deprecated. Do not set. + max_outputs: Optional `int`. At most this many audio clips will be + emitted. When more than `max_outputs` many clips are provided, the + first `max_outputs` many clips will be used and the rest silently + discarded. + encoding: A constant `str` indicating the desired encoding. You + can choose any format you like, as long as it's "wav". Please see + the "API compatibility note" below. + display_name: Optional name for this summary in TensorBoard, as a + `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `tf.Summary` protobuf object. + + API compatibility note: The default value of the `encoding` + argument is _not_ guaranteed to remain unchanged across TensorBoard + versions. In the future, we will by default encode as FLAC instead of + as WAV. If the specific format is important to you, please provide a + file format explicitly. + """ + if labels is not None: + warnings.warn(_LABELS_WARNING) + + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + audio = np.array(audio) + if audio.ndim != 3: + raise ValueError("Shape %r must have rank 3" % (audio.shape,)) + if encoding is None: + encoding = "wav" + + if encoding == "wav": + encoding = metadata.Encoding.Value("WAV") + encoder = functools.partial( + encoder_util.encode_wav, samples_per_second=sample_rate + ) + else: + raise ValueError("Unknown encoding: %r" % encoding) + + limited_audio = audio[:max_outputs] + if labels is None: + limited_labels = [b""] * len(limited_audio) + else: + limited_labels = [ + tf.compat.as_bytes(label) for label in labels[:max_outputs] + ] + + encoded_audio = [encoder(a) for a in limited_audio] + content = np.array([encoded_audio, limited_labels]).transpose() + tensor = tf.make_tensor_proto(content, dtype=tf.string) + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description, encoding=encoding + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + + summary = tf.Summary() + summary.value.add( + tag="%s/audio_summary" % name, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/lib/python3.12/site-packages/tensorboard/plugins/audio/summary_v2.py b/lib/python3.12/site-packages/tensorboard/plugins/audio/summary_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..559bcf8e3b401efc8115c7a610e40e2acb606ec6 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/audio/summary_v2.py @@ -0,0 +1,125 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Audio summaries and TensorFlow operations to create them, V2 versions. + +An audio summary stores a rank-2 string tensor of shape `[k, 2]`, where +`k` is the number of audio clips recorded in the summary. Each row of +the tensor is a pair `[encoded_audio, label]`, where `encoded_audio` is +a binary string whose encoding is specified in the summary metadata, and +`label` is a UTF-8 encoded Markdown string describing the audio clip. +""" + + +import functools + +from tensorboard.compat import tf2 as tf +from tensorboard.plugins.audio import metadata +from tensorboard.util import lazy_tensor_creator + + +def audio( + name, + data, + sample_rate, + step=None, + max_outputs=3, + encoding=None, + description=None, +): + """Write an audio summary. + + Arguments: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + data: A `Tensor` representing audio data with shape `[k, t, c]`, + where `k` is the number of audio clips, `t` is the number of + frames, and `c` is the number of channels. Elements should be + floating-point values in `[-1.0, 1.0]`. Any of the dimensions may + be statically unknown (i.e., `None`). + sample_rate: An `int` or rank-0 `int32` `Tensor` that represents the + sample rate, in Hz. Must be positive. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + max_outputs: Optional `int` or rank-0 integer `Tensor`. At most this + many audio clips will be emitted at each step. When more than + `max_outputs` many clips are provided, the first `max_outputs` + many clips will be used and the rest silently discarded. + encoding: Optional constant `str` for the desired encoding. Only "wav" + is currently supported, but this is not guaranteed to remain the + default, so if you want "wav" in particular, set this explicitly. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + + Returns: + True on success, or false if no summary was emitted because no default + summary writer was available. + + Raises: + ValueError: if a default writer exists, but no step was provided and + `tf.summary.experimental.get_step()` is None. + """ + audio_ops = getattr(tf, "audio", None) + if audio_ops is None: + # Fallback for older versions of TF without tf.audio. + from tensorflow.python.ops import gen_audio_ops as audio_ops + + if encoding is None: + encoding = "wav" + if encoding != "wav": + raise ValueError("Unknown encoding: %r" % encoding) + summary_metadata = metadata.create_summary_metadata( + display_name=None, + description=description, + encoding=metadata.Encoding.Value("WAV"), + ) + inputs = [data, sample_rate, max_outputs, step] + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + with summary_scope(name, "audio_summary", values=inputs) as (tag, _): + # Defer audio encoding preprocessing by passing it as a callable to write(), + # wrapped in a LazyTensorCreator for backwards compatibility, so that we + # only do this work when summaries are actually written. + @lazy_tensor_creator.LazyTensorCreator + def lazy_tensor(): + tf.debugging.assert_rank(data, 3) + tf.debugging.assert_non_negative(max_outputs) + limited_audio = data[:max_outputs] + encode_fn = functools.partial( + audio_ops.encode_wav, sample_rate=sample_rate + ) + encoded_audio = tf.map_fn( + encode_fn, + limited_audio, + dtype=tf.string, + name="encode_each_audio", + ) + # Workaround for map_fn returning float dtype for an empty elems input. + encoded_audio = tf.cond( + tf.shape(input=encoded_audio)[0] > 0, + lambda: encoded_audio, + lambda: tf.constant([], tf.string), + ) + limited_labels = tf.tile([""], tf.shape(input=limited_audio)[:1]) + return tf.transpose(a=tf.stack([encoded_audio, limited_labels])) + + # To ensure that audio encoding logic is only executed when summaries + # are written, we pass callable to `tensor` parameter. + return tf.summary.write( + tag=tag, tensor=lazy_tensor, step=step, metadata=summary_metadata + ) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/base_plugin.py b/lib/python3.12/site-packages/tensorboard/plugins/base_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..f6edeac0205bcd0ae6c544342d9f1ebe8e9dd4c1 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/base_plugin.py @@ -0,0 +1,368 @@ +# Copyright 2016 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorBoard Plugin abstract base class. + +Every plugin in TensorBoard must extend and implement the abstract +methods of this base class. +""" + + +from abc import ABCMeta +from abc import abstractmethod + + +class TBPlugin(metaclass=ABCMeta): + """TensorBoard plugin interface. + + Every plugin must extend from this class. + + Subclasses should have a trivial constructor that takes a TBContext + argument. Any operation that might throw an exception should either be + done lazily or made safe with a TBLoader subclass, so the plugin won't + negatively impact the rest of TensorBoard. + + Fields: + plugin_name: The plugin_name will also be a prefix in the http + handlers, e.g. `data/plugins/$PLUGIN_NAME/$HANDLER` The plugin + name must be unique for each registered plugin, or a ValueError + will be thrown when the application is constructed. The plugin + name must only contain characters among [A-Za-z0-9_.-], and must + be nonempty, or a ValueError will similarly be thrown. + """ + + plugin_name = None + + def __init__(self, context): + """Initializes this plugin. + + The default implementation does nothing. Subclasses are encouraged + to override this and save any necessary fields from the `context`. + + Args: + context: A `base_plugin.TBContext` object. + """ + pass + + @abstractmethod + def get_plugin_apps(self): + """Returns a set of WSGI applications that the plugin implements. + + Each application gets registered with the tensorboard app and is served + under a prefix path that includes the name of the plugin. + + Returns: + A dict mapping route paths to WSGI applications. Each route path + should include a leading slash. + """ + raise NotImplementedError() + + @abstractmethod + def is_active(self): + """Determines whether this plugin is active. + + A plugin may not be active for instance if it lacks relevant data. If a + plugin is inactive, the frontend may avoid issuing requests to its routes. + + Returns: + A boolean value. Whether this plugin is active. + """ + raise NotImplementedError() + + def frontend_metadata(self): + """Defines how the plugin will be displayed on the frontend. + + The base implementation returns a default value. Subclasses + should override this and specify either an `es_module_path` or + (for legacy plugins) an `element_name`, and are encouraged to + set any other relevant attributes. + """ + return FrontendMetadata() + + def data_plugin_names(self): + """Experimental. Lists plugins whose summary data this plugin reads. + + Returns: + A collection of strings representing plugin names (as read + from `SummaryMetadata.plugin_data.plugin_name`) from which + this plugin may read data. Defaults to `(self.plugin_name,)`. + """ + return (self.plugin_name,) + + +class FrontendMetadata: + """Metadata required to render a plugin on the frontend. + + Each argument to the constructor is publicly accessible under a + field of the same name. See constructor docs for further details. + """ + + def __init__( + self, + *, + disable_reload=None, + element_name=None, + es_module_path=None, + remove_dom=None, + tab_name=None, + is_ng_component=None, + ): + """Creates a `FrontendMetadata` value. + + The argument list is sorted and may be extended in the future; + therefore, callers must pass only named arguments to this + constructor. + + Args: + disable_reload: Whether to disable the reload button and + auto-reload timer. A `bool`; defaults to `False`. + element_name: For legacy plugins, name of the custom element + defining the plugin frontend: e.g., `"tf-scalar-dashboard"`. + A `str` or `None` (for iframed plugins). Mutually exclusive + with `es_module_path`. + es_module_path: ES module to use as an entry point to this plugin. + A `str` that is a key in the result of `get_plugin_apps()`, or + `None` for legacy plugins bundled with TensorBoard as part of + `webfiles.zip`. Mutually exclusive with legacy `element_name` + remove_dom: Whether to remove the plugin DOM when switching to a + different plugin, to trigger the Polymer 'detached' event. + A `bool`; defaults to `False`. + tab_name: Name to show in the menu item for this dashboard within + the navigation bar. May differ from the plugin name: for + instance, the tab name should not use underscores to separate + words. Should be a `str` or `None` (the default; indicates to + use the plugin name as the tab name). + is_ng_component: Set to `True` only for built-in Angular plugins. + In this case, the `plugin_name` property of the Plugin, which is + mapped to the `id` property in JavaScript's `UiPluginMetadata` type, + is used to select the Angular component. A `True` value is mutually + exclusive with `element_name` and `es_module_path`. + """ + self._disable_reload = ( + False if disable_reload is None else disable_reload + ) + self._element_name = element_name + self._es_module_path = es_module_path + self._remove_dom = False if remove_dom is None else remove_dom + self._tab_name = tab_name + self._is_ng_component = ( + False if is_ng_component is None else is_ng_component + ) + + @property + def disable_reload(self): + return self._disable_reload + + @property + def element_name(self): + return self._element_name + + @property + def is_ng_component(self): + return self._is_ng_component + + @property + def es_module_path(self): + return self._es_module_path + + @property + def remove_dom(self): + return self._remove_dom + + @property + def tab_name(self): + return self._tab_name + + def __eq__(self, other): + if not isinstance(other, FrontendMetadata): + return False + if self._disable_reload != other._disable_reload: + return False + if self._disable_reload != other._disable_reload: + return False + if self._element_name != other._element_name: + return False + if self._es_module_path != other._es_module_path: + return False + if self._remove_dom != other._remove_dom: + return False + if self._tab_name != other._tab_name: + return False + return True + + def __hash__(self): + return hash( + ( + self._disable_reload, + self._element_name, + self._es_module_path, + self._remove_dom, + self._tab_name, + self._is_ng_component, + ) + ) + + def __repr__(self): + return "FrontendMetadata(%s)" % ", ".join( + ( + "disable_reload=%r" % self._disable_reload, + "element_name=%r" % self._element_name, + "es_module_path=%r" % self._es_module_path, + "remove_dom=%r" % self._remove_dom, + "tab_name=%r" % self._tab_name, + "is_ng_component=%r" % self._is_ng_component, + ) + ) + + +class TBContext: + """Magic container of information passed from TensorBoard core to plugins. + + A TBContext instance is passed to the constructor of a TBPlugin class. Plugins + are strongly encouraged to assume that any of these fields can be None. In + cases when a field is considered mandatory by a plugin, it can either crash + with ValueError, or silently choose to disable itself by returning False from + its is_active method. + + All fields in this object are thread safe. + """ + + def __init__( + self, + *, + assets_zip_provider=None, + data_provider=None, + flags=None, + logdir=None, + multiplexer=None, + plugin_name_to_instance=None, + sampling_hints=None, + window_title=None, + ): + """Instantiates magic container. + + The argument list is sorted and may be extended in the future; therefore, + callers must pass only named arguments to this constructor. + + Args: + assets_zip_provider: A function that returns a newly opened file handle + for a zip file containing all static assets. The file names inside the + zip file are considered absolute paths on the web server. The file + handle this function returns must be closed. It is assumed that you + will pass this file handle to zipfile.ZipFile. This zip file should + also have been created by the tensorboard_zip_file build rule. + data_provider: Instance of `tensorboard.data.provider.DataProvider`. May + be `None` if `flags.generic_data` is set to `"false"`. + flags: An object of the runtime flags provided to TensorBoard to their + values. + logdir: The string logging directory TensorBoard was started with. + multiplexer: An EventMultiplexer with underlying TB data. Plugins should + copy this data over to the database when the db fields are set. + plugin_name_to_instance: A mapping between plugin name to instance. + Plugins may use this property to access other plugins. The context + object is passed to plugins during their construction, so a given + plugin may be absent from this mapping until it is registered. Plugin + logic should handle cases in which a plugin is absent from this + mapping, lest a KeyError is raised. + sampling_hints: Map from plugin name to `int` or `NoneType`, where + the value represents the user-specified downsampling limit as + given to the `--samples_per_plugin` flag, or `None` if none was + explicitly given for this plugin. + window_title: A string specifying the window title. + """ + self.assets_zip_provider = assets_zip_provider + self.data_provider = data_provider + self.flags = flags + self.logdir = logdir + self.multiplexer = multiplexer + self.plugin_name_to_instance = plugin_name_to_instance + self.sampling_hints = sampling_hints + self.window_title = window_title + + +class TBLoader: + """TBPlugin factory base class. + + Plugins can override this class to customize how a plugin is loaded at + startup. This might entail adding command-line arguments, checking if + optional dependencies are installed, and potentially also specializing + the plugin class at runtime. + + When plugins use optional dependencies, the loader needs to be + specified in its own module. That way it's guaranteed to be + importable, even if the `TBPlugin` itself can't be imported. + + Subclasses must have trivial constructors. + """ + + def define_flags(self, parser): + """Adds plugin-specific CLI flags to parser. + + The default behavior is to do nothing. + + When overriding this method, it's recommended that plugins call the + `parser.add_argument_group(plugin_name)` method for readability. No + flags should be specified that would cause `parse_args([])` to fail. + + Args: + parser: The argument parsing object, which may be mutated. + """ + pass + + def fix_flags(self, flags): + """Allows flag values to be corrected or validated after parsing. + + Args: + flags: The parsed argparse.Namespace object. + + Raises: + base_plugin.FlagsError: If a flag is invalid or a required + flag is not passed. + """ + pass + + def load(self, context): + """Loads a TBPlugin instance during the setup phase. + + Args: + context: The TBContext instance. + + Returns: + A plugin instance or None if it could not be loaded. Loaders that return + None are skipped. + + :type context: TBContext + :rtype: TBPlugin | None + """ + return None + + +class BasicLoader(TBLoader): + """Simple TBLoader that's sufficient for most plugins.""" + + def __init__(self, plugin_class): + """Creates simple plugin instance maker. + + :param plugin_class: :class:`TBPlugin` + """ + self.plugin_class = plugin_class + + def load(self, context): + return self.plugin_class(context) + + +class FlagsError(ValueError): + """Raised when a command line flag is not specified or is invalid.""" + + pass diff --git a/lib/python3.12/site-packages/tensorboard/plugins/core/__init__.py b/lib/python3.12/site-packages/tensorboard/plugins/core/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/plugins/core/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/core/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da7242241ac38d320a804c6f68d8f8adb4d2e85a Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/core/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/core/__pycache__/core_plugin.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/core/__pycache__/core_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d46485f3bdff7455f846b2ab1706daf08971a60 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/core/__pycache__/core_plugin.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/core/core_plugin.py b/lib/python3.12/site-packages/tensorboard/plugins/core/core_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..3ef86419f8fae2d3e7e183ef3ee7c7ea247d1289 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/core/core_plugin.py @@ -0,0 +1,755 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""TensorBoard core plugin package.""" + + +import argparse +import functools +import gzip +import io +import mimetypes +import posixpath +import zipfile + +from werkzeug import utils +from werkzeug import wrappers + +from tensorboard import plugin_util +from tensorboard.backend import http_util +from tensorboard.plugins import base_plugin +from tensorboard.util import grpc_util +from tensorboard.util import tb_logging +from tensorboard import version + +logger = tb_logging.get_logger() + + +# If no port is specified, try to bind to this port. See help for --port +# for more details. +DEFAULT_PORT = 6006 +# Valid javascript mimetypes that we have seen configured, in practice. +# Historically (~2020-2022) we saw "application/javascript" exclusively but with +# RFC 9239 (https://www.rfc-editor.org/rfc/rfc9239) we saw some systems quickly +# transition to 'text/javascript'. +JS_MIMETYPES = ["text/javascript", "application/javascript"] +JS_CACHE_EXPIRATION_IN_SECS = 86400 + + +class CorePlugin(base_plugin.TBPlugin): + """Core plugin for TensorBoard. + + This plugin serves runs, configuration data, and static assets. This + plugin should always be present in a TensorBoard WSGI application. + """ + + plugin_name = "core" + + def __init__(self, context, include_debug_info=None): + """Instantiates CorePlugin. + + Args: + context: A base_plugin.TBContext instance. + include_debug_info: If true, `/data/environment` will include some + basic information like the TensorBoard server version. Disabled by + default to prevent surprising information leaks in custom builds of + TensorBoard. + """ + self._flags = context.flags + logdir_spec = context.flags.logdir_spec if context.flags else "" + self._logdir = context.logdir or logdir_spec + self._window_title = context.window_title + self._path_prefix = context.flags.path_prefix if context.flags else None + self._assets_zip_provider = context.assets_zip_provider + self._data_provider = context.data_provider + self._include_debug_info = bool(include_debug_info) + + def is_active(self): + return True + + def get_plugin_apps(self): + apps = { + "/___rPc_sWiTcH___": self._send_404_without_logging, + "/audio": self._redirect_to_index, + "/data/environment": self._serve_environment, + "/data/logdir": self._serve_logdir, + "/data/runs": self._serve_runs, + "/data/experiments": self._serve_experiments, + "/data/experiment_runs": self._serve_experiment_runs, + "/data/notifications": self._serve_notifications, + "/data/window_properties": self._serve_window_properties, + "/events": self._redirect_to_index, + "/favicon.ico": self._send_404_without_logging, + "/graphs": self._redirect_to_index, + "/histograms": self._redirect_to_index, + "/images": self._redirect_to_index, + } + apps.update(self.get_resource_apps()) + return apps + + def get_resource_apps(self): + apps = {} + if not self._assets_zip_provider: + return apps + + with self._assets_zip_provider() as fp: + with zipfile.ZipFile(fp) as zip_: + for path in zip_.namelist(): + content = zip_.read(path) + # Opt out of gzipping index.html + if path == "index.html": + apps["/" + path] = functools.partial( + self._serve_index, content + ) + continue + + gzipped_asset_bytes = _gzip(content) + wsgi_app = functools.partial( + self._serve_asset, path, gzipped_asset_bytes + ) + apps["/" + path] = wsgi_app + apps["/"] = apps["/index.html"] + return apps + + @wrappers.Request.application + def _send_404_without_logging(self, request): + return http_util.Respond(request, "Not found", "text/plain", code=404) + + @wrappers.Request.application + def _redirect_to_index(self, unused_request): + return utils.redirect("/") + + @wrappers.Request.application + def _serve_asset(self, path, gzipped_asset_bytes, request): + """Serves a pre-gzipped static asset from the zip file.""" + mimetype = mimetypes.guess_type(path)[0] or "application/octet-stream" + + # Cache JS resources while keep others do not cache. + expires = ( + JS_CACHE_EXPIRATION_IN_SECS + if request.args.get("_file_hash") and mimetype in JS_MIMETYPES + else 0 + ) + + return http_util.Respond( + request, + gzipped_asset_bytes, + mimetype, + content_encoding="gzip", + expires=expires, + ) + + @wrappers.Request.application + def _serve_index(self, index_asset_bytes, request): + """Serves index.html content. + + Note that we opt out of gzipping index.html to write preamble before the + resource content. This inflates the resource size from 2x kiB to 1xx + kiB, but we require an ability to flush preamble with the HTML content. + """ + relpath = ( + posixpath.relpath(self._path_prefix, request.script_root) + if self._path_prefix + else "." + ) + meta_header = ( + '' + % relpath + ) + content = meta_header.encode("utf-8") + index_asset_bytes + # By passing content_encoding, disallow gzipping. Bloats the content + # from ~25 kiB to ~120 kiB but reduces CPU usage and avoid 3ms worth of + # gzipping. + return http_util.Respond( + request, content, "text/html", content_encoding="identity" + ) + + @wrappers.Request.application + def _serve_environment(self, request): + """Serve a JSON object describing the TensorBoard parameters.""" + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + md = self._data_provider.experiment_metadata( + ctx, experiment_id=experiment + ) + + environment = { + "version": version.VERSION, + "data_location": md.data_location, + "window_title": self._window_title, + "experiment_name": md.experiment_name, + "experiment_description": md.experiment_description, + "creation_time": md.creation_time, + } + if self._include_debug_info: + environment["debug"] = { + "data_provider": str(self._data_provider), + "flags": self._render_flags(), + } + return http_util.Respond( + request, + environment, + "application/json", + ) + + def _render_flags(self): + """Return a JSON-and-human-friendly version of `self._flags`. + + Like `json.loads(json.dumps(self._flags, default=str))` but + without the wasteful serialization overhead. + """ + if self._flags is None: + return None + + def go(x): + if isinstance(x, (type(None), str, int, float)): + return x + if isinstance(x, (list, tuple)): + return [go(v) for v in x] + if isinstance(x, dict): + return {str(k): go(v) for (k, v) in x.items()} + return str(x) + + return go(vars(self._flags)) + + @wrappers.Request.application + def _serve_logdir(self, request): + """Respond with a JSON object containing this TensorBoard's logdir.""" + # TODO(chihuahua): Remove this method once the frontend instead uses the + # /data/environment route (and no deps throughout Google use the + # /data/logdir route). + return http_util.Respond( + request, {"logdir": self._logdir}, "application/json" + ) + + @wrappers.Request.application + def _serve_window_properties(self, request): + """Serve a JSON object containing this TensorBoard's window + properties.""" + # TODO(chihuahua): Remove this method once the frontend instead uses the + # /data/environment route. + return http_util.Respond( + request, {"window_title": self._window_title}, "application/json" + ) + + @wrappers.Request.application + def _serve_runs(self, request): + """Serve a JSON array of run names, ordered by run started time. + + Sort order is by started time (aka first event time) with empty + times sorted last, and then ties are broken by sorting on the + run name. + """ + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + runs = sorted( + self._data_provider.list_runs(ctx, experiment_id=experiment), + key=lambda run: ( + run.start_time if run.start_time is not None else float("inf"), + run.run_name, + ), + ) + run_names = [run.run_name for run in runs] + return http_util.Respond(request, run_names, "application/json") + + @wrappers.Request.application + def _serve_experiments(self, request): + """Serve a JSON array of experiments. + + Experiments are ordered by experiment started time (aka first + event time) with empty times sorted last, and then ties are + broken by sorting on the experiment name. + """ + results = self.list_experiments_impl() + return http_util.Respond(request, results, "application/json") + + def list_experiments_impl(self): + return [] + + @wrappers.Request.application + def _serve_experiment_runs(self, request): + """Serve a JSON runs of an experiment, specified with query param + `experiment`, with their nested data, tag, populated. + + Runs returned are ordered by started time (aka first event time) + with empty times sorted last, and then ties are broken by + sorting on the run name. Tags are sorted by its name, + displayName, and lastly, inserted time. + """ + results = [] + return http_util.Respond(request, results, "application/json") + + @wrappers.Request.application + def _serve_notifications(self, request): + """Serve JSON payload of notifications to show in the UI.""" + response = utils.redirect("../notifications_note.json") + # Disable Werkzeug's automatic Location header correction routine, which + # absolutizes relative paths "to be RFC conformant" [1], but this is + # based on an outdated HTTP/1.1 RFC; the current one allows them: + # https://tools.ietf.org/html/rfc7231#section-7.1.2 + response.autocorrect_location_header = False + return response + + +class CorePluginLoader(base_plugin.TBLoader): + """CorePlugin factory.""" + + def __init__(self, include_debug_info=None): + self._include_debug_info = include_debug_info + + def define_flags(self, parser): + """Adds standard TensorBoard CLI flags to parser.""" + parser.add_argument( + "--logdir", + metavar="PATH", + type=str, + default="", + help="""\ +Directory where TensorBoard will look to find TensorFlow event files +that it can display. TensorBoard will recursively walk the directory +structure rooted at logdir, looking for .*tfevents.* files. + +A leading tilde will be expanded with the semantics of Python's +os.expanduser function. +""", + ) + + parser.add_argument( + "--logdir_spec", + metavar="PATH_SPEC", + type=str, + default="", + help="""\ +Like `--logdir`, but with special interpretation for commas and colons: +commas separate multiple runs, where a colon specifies a new name for a +run. For example: +`tensorboard --logdir_spec=name1:/path/to/logs/1,name2:/path/to/logs/2`. + +This flag is discouraged and can usually be avoided. TensorBoard walks +log directories recursively; for finer-grained control, prefer using a +symlink tree. Some features may not work when using `--logdir_spec` +instead of `--logdir`. +""", + ) + + parser.add_argument( + "--host", + metavar="ADDR", + type=str, + default=None, # like localhost, but prints a note about `--bind_all` + help="""\ +What host to listen to (default: localhost). To serve to the entire local +network on both IPv4 and IPv6, see `--bind_all`, with which this option is +mutually exclusive. +""", + ) + + parser.add_argument( + "--bind_all", + action="store_true", + help="""\ +Serve on all public interfaces. This will expose your TensorBoard instance to +the network on both IPv4 and IPv6 (where available). Mutually exclusive with +`--host`. +""", + ) + + parser.add_argument( + "--port", + metavar="PORT", + type=lambda s: (None if s == "default" else int(s)), + default="default", + help="""\ +Port to serve TensorBoard on. Pass 0 to request an unused port selected +by the operating system, or pass "default" to try to bind to the default +port (%s) but search for a nearby free port if the default port is +unavailable. (default: "default").\ +""" + % DEFAULT_PORT, + ) + + parser.add_argument( + "--reuse_port", + metavar="BOOL", + # Custom str-to-bool converter since regular bool() doesn't work. + type=lambda v: {"true": True, "false": False}.get(v.lower(), v), + choices=[True, False], + default=False, + help="""\ +Enables the SO_REUSEPORT option on the socket opened by TensorBoard's HTTP +server, for platforms that support it. This is useful in cases when a parent +process has obtained the port already and wants to delegate access to the +port to TensorBoard as a subprocess.(default: %(default)s).\ +""", + ) + + parser.add_argument( + "--load_fast", + type=str, + default="auto", + choices=["false", "auto", "true"], + help="""\ +Use alternate mechanism to load data. Typically 100x faster or more, but only +available on some platforms and invocations. Defaults to "auto" to use this new +mode only if available, otherwise falling back to the legacy loading path. Set +to "true" to suppress the advisory note and hard-fail if the fast codepath is +not available. Set to "false" to always fall back. Feedback/issues: +https://github.com/tensorflow/tensorboard/issues/4784 +(default: %(default)s) +""", + ) + + parser.add_argument( + "--extra_data_server_flags", + type=str, + default="", + help="""\ +Experimental. With `--load_fast`, pass these additional command-line flags to +the data server. Subject to POSIX word splitting per `shlex.split`. Meant for +debugging; not officially supported. +""", + ) + + parser.add_argument( + "--grpc_creds_type", + type=grpc_util.ChannelCredsType, + default=grpc_util.ChannelCredsType.LOCAL, + choices=grpc_util.ChannelCredsType.choices(), + help="""\ +Experimental. The type of credentials to use to connect to the data server. +(default: %(default)s) +""", + ) + + parser.add_argument( + "--grpc_data_provider", + metavar="PORT", + type=str, + default="", + help="""\ +Experimental. Address of a gRPC server exposing a data provider. Set to empty +string to disable. (default: %(default)s) +""", + ) + + parser.add_argument( + "--purge_orphaned_data", + metavar="BOOL", + # Custom str-to-bool converter since regular bool() doesn't work. + type=lambda v: {"true": True, "false": False}.get(v.lower(), v), + choices=[True, False], + default=True, + help="""\ +Whether to purge data that may have been orphaned due to TensorBoard +restarts. Setting --purge_orphaned_data=False can be used to debug data +disappearance. (default: %(default)s)\ +""", + ) + + parser.add_argument( + "--db", + metavar="URI", + type=str, + default="", + help="""\ +[experimental] sets SQL database URI and enables DB backend mode, which is +read-only unless --db_import is also passed.\ +""", + ) + + parser.add_argument( + "--db_import", + action="store_true", + help="""\ +[experimental] enables DB read-and-import mode, which in combination with +--logdir imports event files into a DB backend on the fly. The backing DB is +temporary unless --db is also passed to specify a DB path to use.\ +""", + ) + + parser.add_argument( + "--inspect", + action="store_true", + help="""\ +Prints digests of event files to command line. + +This is useful when no data is shown on TensorBoard, or the data shown +looks weird. + +Must specify one of `logdir` or `event_file` flag. + +Example usage: + `tensorboard --inspect --logdir mylogdir --tag loss` + +See tensorboard/backend/event_processing/event_file_inspector.py for more info.\ +""", + ) + + # This flag has a "_tb" suffix to avoid conflicting with an internal flag + # named --version. Note that due to argparse auto-expansion of unambiguous + # flag prefixes, you can still invoke this as `tensorboard --version`. + parser.add_argument( + "--version_tb", + action="store_true", + help="Prints the version of Tensorboard", + ) + + parser.add_argument( + "--tag", + metavar="TAG", + type=str, + default="", + help="tag to query for; used with --inspect", + ) + + parser.add_argument( + "--event_file", + metavar="PATH", + type=str, + default="", + help="""\ +The particular event file to query for. Only used if --inspect is +present and --logdir is not specified.\ +""", + ) + + parser.add_argument( + "--path_prefix", + metavar="PATH", + type=str, + default="", + help="""\ +An optional, relative prefix to the path, e.g. "/path/to/tensorboard". +resulting in the new base url being located at +localhost:6006/path/to/tensorboard under default settings. A leading +slash is required when specifying the path_prefix. A trailing slash is +optional and has no effect. The path_prefix can be leveraged for path +based routing of an ELB when the website base_url is not available e.g. +"example.site.com/path/to/tensorboard/".\ +""", + ) + + parser.add_argument( + "--window_title", + metavar="TEXT", + type=str, + default="", + help="changes title of browser window", + ) + + parser.add_argument( + "--max_reload_threads", + metavar="COUNT", + type=int, + default=1, + help="""\ +The max number of threads that TensorBoard can use to reload runs. Not +relevant for db read-only mode. Each thread reloads one run at a time. +(default: %(default)s)\ +""", + ) + + parser.add_argument( + "--reload_interval", + metavar="SECONDS", + type=_nonnegative_float, + default=5.0, + help="""\ +How often the backend should load more data, in seconds. Set to 0 to +load just once at startup. Must be non-negative. (default: %(default)s)\ +""", + ) + + parser.add_argument( + "--reload_task", + metavar="TYPE", + type=str, + default="auto", + choices=["auto", "thread", "process", "blocking"], + help="""\ +[experimental] The mechanism to use for the background data reload task. +The default "auto" option will conditionally use threads for legacy reloading +and a child process for DB import reloading. The "process" option is only +useful with DB import mode. The "blocking" option will block startup until +reload finishes, and requires --load_interval=0. (default: %(default)s)\ +""", + ) + + parser.add_argument( + "--reload_multifile", + metavar="BOOL", + # Custom str-to-bool converter since regular bool() doesn't work. + type=lambda v: {"true": True, "false": False}.get(v.lower(), v), + choices=[True, False], + default=None, + help="""\ +[experimental] If true, this enables experimental support for continuously +polling multiple event files in each run directory for newly appended data +(rather than only polling the last event file). Event files will only be +polled as long as their most recently read data is newer than the threshold +defined by --reload_multifile_inactive_secs, to limit resource usage. Beware +of running out of memory if the logdir contains many active event files. +(default: false)\ +""", + ) + + parser.add_argument( + "--reload_multifile_inactive_secs", + metavar="SECONDS", + type=int, + default=86400, + help="""\ +[experimental] Configures the age threshold in seconds at which an event file +that has no event wall time more recent than that will be considered an +inactive file and no longer polled (to limit resource usage). If set to -1, +no maximum age will be enforced, but beware of running out of memory and +heavier filesystem read traffic. If set to 0, this reverts to the older +last-file-only polling strategy (akin to --reload_multifile=false). +(default: %(default)s - intended to ensure an event file remains active if +it receives new data at least once per 24 hour period)\ +""", + ) + + parser.add_argument( + "--generic_data", + metavar="TYPE", + type=str, + default="auto", + choices=["false", "auto", "true"], + help="""\ +[experimental] Hints whether plugins should read from generic data +provider infrastructure. For plugins that support only the legacy +multiplexer APIs or only the generic data APIs, this option has no +effect. The "auto" option enables this only for plugins that are +considered to have stable support for generic data providers. (default: +%(default)s)\ +""", + ) + + parser.add_argument( + "--samples_per_plugin", + type=_parse_samples_per_plugin, + default="", + help="""\ +An optional comma separated list of plugin_name=num_samples pairs to +explicitly specify how many samples to keep per tag for that plugin. For +unspecified plugins, TensorBoard randomly downsamples logged summaries +to reasonable values to prevent out-of-memory errors for long running +jobs. This flag allows fine control over that downsampling. Note that if a +plugin is not specified in this list, a plugin-specific default number of +samples will be enforced. (for example, 10 for images, 500 for histograms, +and 1000 for scalars). Most users should not need to set this flag.\ +""", + ) + + parser.add_argument( + "--detect_file_replacement", + metavar="BOOL", + # Custom str-to-bool converter since regular bool() doesn't work. + type=lambda v: {"true": True, "false": False}.get(v.lower(), v), + choices=[True, False], + default=None, + help="""\ +[experimental] If true, this enables experimental support for detecting when +event files are replaced with new versions that contain additional data. This is +not needed in the normal case where new data is either appended to an existing +file or written to a brand new file, but it arises, for example, when using +rsync without the --inplace option, in which new versions of the original file +are first written to a temporary file, then swapped into the final location. + +This option is currently incompatible with --load_fast=true, and if passed will +disable fast-loading mode. (default: false)\ +""", + ) + + def fix_flags(self, flags): + """Fixes standard TensorBoard CLI flags to parser.""" + FlagsError = base_plugin.FlagsError + if flags.version_tb: + pass + elif flags.inspect: + if flags.logdir_spec: + raise FlagsError( + "--logdir_spec is not supported with --inspect." + ) + if flags.logdir and flags.event_file: + raise FlagsError( + "Must specify either --logdir or --event_file, but not both." + ) + if not (flags.logdir or flags.event_file): + raise FlagsError( + "Must specify either --logdir or --event_file." + ) + elif flags.logdir and flags.logdir_spec: + raise FlagsError("May not specify both --logdir and --logdir_spec") + elif ( + not flags.db + and not flags.logdir + and not flags.logdir_spec + and not flags.grpc_data_provider + ): + raise FlagsError( + "A logdir or db must be specified. " + "For example `tensorboard --logdir mylogdir` " + "or `tensorboard --db sqlite:~/.tensorboard.db`. " + "Run `tensorboard --helpfull` for details and examples." + ) + elif flags.host is not None and flags.bind_all: + raise FlagsError("Must not specify both --host and --bind_all.") + elif ( + flags.load_fast == "true" and flags.detect_file_replacement is True + ): + raise FlagsError( + "Must not specify both --load_fast=true and" + "--detect_file_replacement=true" + ) + + flags.path_prefix = flags.path_prefix.rstrip("/") + if flags.path_prefix and not flags.path_prefix.startswith("/"): + raise FlagsError( + "Path prefix must start with slash, but got: %r." + % flags.path_prefix + ) + + def load(self, context): + """Creates CorePlugin instance.""" + return CorePlugin(context, include_debug_info=self._include_debug_info) + + +def _gzip(bytestring): + out = io.BytesIO() + # Set mtime to zero for deterministic results across TensorBoard launches. + with gzip.GzipFile(fileobj=out, mode="wb", compresslevel=3, mtime=0) as f: + f.write(bytestring) + return out.getvalue() + + +def _parse_samples_per_plugin(value): + """Parses `value` as a string-to-int dict in the form `foo=12,bar=34`.""" + result = {} + for token in value.split(","): + if token: + k, v = token.strip().split("=") + result[k] = int(v) + return result + + +def _nonnegative_float(v): + try: + v = float(v) + except ValueError: + raise argparse.ArgumentTypeError("invalid float: %r" % v) + if not (v >= 0): # no NaNs, please + raise argparse.ArgumentTypeError("must be non-negative: %r" % v) + return v diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__init__.py b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..788c17187317af6ed0bb676928c4f719f66dcc0e Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/custom_scalars_plugin.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/custom_scalars_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fca1a5a3435fb1d8de11f0420cf8b044a250a568 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/custom_scalars_plugin.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/layout_pb2.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/layout_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e3aa0887e34c3bd69d0076d340f30d4d4f5035a0 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/layout_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/metadata.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2a17d5348c49e219f786370d48e1d74fcecad83 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/metadata.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/summary.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/summary.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..128fcb04613c585b67e9da46616e1906c066f078 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/__pycache__/summary.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/custom_scalars_plugin.py b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/custom_scalars_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..423593bbb0ec5a5fb8c8c929bbbe0839c2dea873 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/custom_scalars_plugin.py @@ -0,0 +1,321 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The TensorBoard Custom Scalars plugin. + +This plugin lets the user create scalars plots with custom run-tag combinations +by specifying regular expressions. + +See `http_api.md` in this directory for specifications of the routes for +this plugin. +""" + +import re + +from werkzeug import wrappers + +from tensorboard import plugin_util +from tensorboard.backend import http_util +from tensorboard.compat import tf +from tensorboard.data import provider +from tensorboard.plugins import base_plugin +from tensorboard.plugins.custom_scalar import layout_pb2 +from tensorboard.plugins.custom_scalar import metadata +from tensorboard.plugins.scalar import metadata as scalars_metadata +from tensorboard.plugins.scalar import scalars_plugin + + +# The name of the property in the response for whether the regex is valid. +_REGEX_VALID_PROPERTY = "regex_valid" + +# The name of the property in the response for the payload (tag to ScalarEvents +# mapping). +_TAG_TO_EVENTS_PROPERTY = "tag_to_events" + +# The number of seconds to wait in between checks for the config file specifying +# layout. +_CONFIG_FILE_CHECK_THROTTLE = 60 + + +class CustomScalarsPlugin(base_plugin.TBPlugin): + """CustomScalars Plugin for TensorBoard.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates ScalarsPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._logdir = context.logdir + self._data_provider = context.data_provider + self._plugin_name_to_instance = context.plugin_name_to_instance + + def _get_scalars_plugin(self): + """Tries to get the scalars plugin. + + Returns: + The scalars plugin. Or None if it is not yet registered. + """ + if scalars_metadata.PLUGIN_NAME in self._plugin_name_to_instance: + # The plugin is registered. + return self._plugin_name_to_instance[scalars_metadata.PLUGIN_NAME] + # The plugin is not yet registered. + return None + + def get_plugin_apps(self): + return { + "/download_data": self.download_data_route, + "/layout": self.layout_route, + "/scalars": self.scalars_route, + } + + def is_active(self): + """Plugin is active if there is a custom layout for the dashboard.""" + return False # `list_plugins` as called by TB core suffices + + def frontend_metadata(self): + return base_plugin.FrontendMetadata( + element_name="tf-custom-scalar-dashboard", + tab_name="Custom Scalars", + ) + + @wrappers.Request.application + def download_data_route(self, request): + ctx = plugin_util.context(request.environ) + run = request.args.get("run") + tag = request.args.get("tag") + experiment = plugin_util.experiment_id(request.environ) + response_format = request.args.get("format") + try: + body, mime_type = self.download_data_impl( + ctx, run, tag, experiment, response_format + ) + except ValueError as e: + return http_util.Respond( + request=request, + content=str(e), + content_type="text/plain", + code=400, + ) + return http_util.Respond(request, body, mime_type) + + def download_data_impl(self, ctx, run, tag, experiment, response_format): + """Provides a response for downloading scalars data for a data series. + + Args: + ctx: A tensorboard.context.RequestContext value. + run: The run. + tag: The specific tag. + experiment: An experiment ID, as a possibly-empty `str`. + response_format: A string. One of the values of the OutputFormat enum + of the scalar plugin. + + Raises: + ValueError: If the scalars plugin is not registered. + + Returns: + 2 entities: + - A JSON object response body. + - A mime type (string) for the response. + """ + scalars_plugin_instance = self._get_scalars_plugin() + if not scalars_plugin_instance: + raise ValueError( + ( + "Failed to respond to request for /download_data. " + "The scalars plugin is oddly not registered." + ) + ) + + body, mime_type = scalars_plugin_instance.scalars_impl( + ctx, tag, run, experiment, response_format + ) + return body, mime_type + + @wrappers.Request.application + def scalars_route(self, request): + """Given a tag regex and single run, return ScalarEvents. + + This route takes 2 GET params: + run: A run string to find tags for. + tag: A string that is a regex used to find matching tags. + The response is a JSON object: + { + // Whether the regular expression is valid. Also false if empty. + regexValid: boolean, + + // An object mapping tag name to a list of ScalarEvents. + payload: Object, + } + """ + ctx = plugin_util.context(request.environ) + tag_regex_string = request.args.get("tag") + run = request.args.get("run") + experiment = plugin_util.experiment_id(request.environ) + mime_type = "application/json" + + try: + body = self.scalars_impl(ctx, run, tag_regex_string, experiment) + except ValueError as e: + return http_util.Respond( + request=request, + content=str(e), + content_type="text/plain", + code=400, + ) + + # Produce the response. + return http_util.Respond(request, body, mime_type) + + def scalars_impl(self, ctx, run, tag_regex_string, experiment): + """Given a tag regex and single run, return ScalarEvents. + + Args: + ctx: A tensorboard.context.RequestContext value. + run: A run string. + tag_regex_string: A regular expression that captures portions of tags. + + Raises: + ValueError: if the scalars plugin is not registered. + + Returns: + A dictionary that is the JSON-able response. + """ + if not tag_regex_string: + # The user provided no regex. + return { + _REGEX_VALID_PROPERTY: False, + _TAG_TO_EVENTS_PROPERTY: {}, + } + + # Construct the regex. + try: + regex = re.compile(tag_regex_string) + except re.error: + return { + _REGEX_VALID_PROPERTY: False, + _TAG_TO_EVENTS_PROPERTY: {}, + } + + # Fetch the tags for the run. Filter for tags that match the regex. + run_to_data = self._data_provider.list_scalars( + ctx, + experiment_id=experiment, + plugin_name=scalars_metadata.PLUGIN_NAME, + run_tag_filter=provider.RunTagFilter(runs=[run]), + ) + + tag_to_data = None + try: + tag_to_data = run_to_data[run] + except KeyError: + # The run could not be found. Perhaps a configuration specified a run that + # TensorBoard has not read from disk yet. + payload = {} + + if tag_to_data: + scalars_plugin_instance = self._get_scalars_plugin() + if not scalars_plugin_instance: + raise ValueError( + ( + "Failed to respond to request for /scalars. " + "The scalars plugin is oddly not registered." + ) + ) + + form = scalars_plugin.OutputFormat.JSON + payload = { + tag: scalars_plugin_instance.scalars_impl( + ctx, tag, run, experiment, form + )[0] + for tag in tag_to_data.keys() + if regex.match(tag) + } + + return { + _REGEX_VALID_PROPERTY: True, + _TAG_TO_EVENTS_PROPERTY: payload, + } + + @wrappers.Request.application + def layout_route(self, request): + """Fetches the custom layout specified by the config file in the logdir. + + If more than 1 run contains a layout, this method merges the layouts by + merging charts within individual categories. If 2 categories with the same + name are found, the charts within are merged. The merging is based on the + order of the runs to which the layouts are written. + + The response is a JSON object mirroring properties of the Layout proto if a + layout for any run is found. + + The response is an empty object if no layout could be found. + """ + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + body = self.layout_impl(ctx, experiment) + return http_util.Respond(request, body, "application/json") + + def layout_impl(self, ctx, experiment): + # Keep a mapping between and category so we do not create duplicate + # categories. + title_to_category = {} + + merged_layout = None + data = self._data_provider.read_tensors( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + run_tag_filter=provider.RunTagFilter( + tags=[metadata.CONFIG_SUMMARY_TAG] + ), + downsample=1, + ) + for run in sorted(data): + points = data[run][metadata.CONFIG_SUMMARY_TAG] + content = points[0].numpy.item() + layout_proto = layout_pb2.Layout() + layout_proto.ParseFromString(tf.compat.as_bytes(content)) + + if merged_layout: + # Append the categories within this layout to the merged layout. + for category in layout_proto.category: + if category.title in title_to_category: + # A category with this name has been seen before. Do not create a + # new one. Merge their charts, skipping any duplicates. + title_to_category[category.title].chart.extend( + [ + c + for c in category.chart + if c + not in title_to_category[category.title].chart + ] + ) + else: + # This category has not been seen before. + merged_layout.category.add().MergeFrom(category) + title_to_category[category.title] = category + else: + # This is the first layout encountered. + merged_layout = layout_proto + for category in layout_proto.category: + title_to_category[category.title] = category + + if merged_layout: + return plugin_util.proto_to_json(merged_layout) + else: + # No layout was found. + return {} diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/layout_pb2.py b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/layout_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..1c511cf336039b89c6d19d3fc534bbc5667e07ed --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/layout_pb2.py @@ -0,0 +1,85 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorboard/plugins/custom_scalar/layout.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.tensorboard/plugins/custom_scalar/layout.proto\x12\x0btensorboard\"\x8d\x01\n\x05\x43hart\x12\r\n\x05title\x18\x01 \x01(\t\x12\x37\n\tmultiline\x18\x02 \x01(\x0b\x32\".tensorboard.MultilineChartContentH\x00\x12\x31\n\x06margin\x18\x03 \x01(\x0b\x32\x1f.tensorboard.MarginChartContentH\x00\x42\t\n\x07\x63ontent\"$\n\x15MultilineChartContent\x12\x0b\n\x03tag\x18\x01 \x03(\t\"\x83\x01\n\x12MarginChartContent\x12\x36\n\x06series\x18\x01 \x03(\x0b\x32&.tensorboard.MarginChartContent.Series\x1a\x35\n\x06Series\x12\r\n\x05value\x18\x01 \x01(\t\x12\r\n\x05lower\x18\x02 \x01(\t\x12\r\n\x05upper\x18\x03 \x01(\t\"L\n\x08\x43\x61tegory\x12\r\n\x05title\x18\x01 \x01(\t\x12!\n\x05\x63hart\x18\x02 \x03(\x0b\x32\x12.tensorboard.Chart\x12\x0e\n\x06\x63losed\x18\x03 \x01(\x08\"B\n\x06Layout\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\'\n\x08\x63\x61tegory\x18\x02 \x03(\x0b\x32\x15.tensorboard.Categoryb\x06proto3') + + + +_CHART = DESCRIPTOR.message_types_by_name['Chart'] +_MULTILINECHARTCONTENT = DESCRIPTOR.message_types_by_name['MultilineChartContent'] +_MARGINCHARTCONTENT = DESCRIPTOR.message_types_by_name['MarginChartContent'] +_MARGINCHARTCONTENT_SERIES = _MARGINCHARTCONTENT.nested_types_by_name['Series'] +_CATEGORY = DESCRIPTOR.message_types_by_name['Category'] +_LAYOUT = DESCRIPTOR.message_types_by_name['Layout'] +Chart = _reflection.GeneratedProtocolMessageType('Chart', (_message.Message,), { + 'DESCRIPTOR' : _CHART, + '__module__' : 'tensorboard.plugins.custom_scalar.layout_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.Chart) + }) +_sym_db.RegisterMessage(Chart) + +MultilineChartContent = _reflection.GeneratedProtocolMessageType('MultilineChartContent', (_message.Message,), { + 'DESCRIPTOR' : _MULTILINECHARTCONTENT, + '__module__' : 'tensorboard.plugins.custom_scalar.layout_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.MultilineChartContent) + }) +_sym_db.RegisterMessage(MultilineChartContent) + +MarginChartContent = _reflection.GeneratedProtocolMessageType('MarginChartContent', (_message.Message,), { + + 'Series' : _reflection.GeneratedProtocolMessageType('Series', (_message.Message,), { + 'DESCRIPTOR' : _MARGINCHARTCONTENT_SERIES, + '__module__' : 'tensorboard.plugins.custom_scalar.layout_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.MarginChartContent.Series) + }) + , + 'DESCRIPTOR' : _MARGINCHARTCONTENT, + '__module__' : 'tensorboard.plugins.custom_scalar.layout_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.MarginChartContent) + }) +_sym_db.RegisterMessage(MarginChartContent) +_sym_db.RegisterMessage(MarginChartContent.Series) + +Category = _reflection.GeneratedProtocolMessageType('Category', (_message.Message,), { + 'DESCRIPTOR' : _CATEGORY, + '__module__' : 'tensorboard.plugins.custom_scalar.layout_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.Category) + }) +_sym_db.RegisterMessage(Category) + +Layout = _reflection.GeneratedProtocolMessageType('Layout', (_message.Message,), { + 'DESCRIPTOR' : _LAYOUT, + '__module__' : 'tensorboard.plugins.custom_scalar.layout_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.Layout) + }) +_sym_db.RegisterMessage(Layout) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _CHART._serialized_start=64 + _CHART._serialized_end=205 + _MULTILINECHARTCONTENT._serialized_start=207 + _MULTILINECHARTCONTENT._serialized_end=243 + _MARGINCHARTCONTENT._serialized_start=246 + _MARGINCHARTCONTENT._serialized_end=377 + _MARGINCHARTCONTENT_SERIES._serialized_start=324 + _MARGINCHARTCONTENT_SERIES._serialized_end=377 + _CATEGORY._serialized_start=379 + _CATEGORY._serialized_end=455 + _LAYOUT._serialized_start=457 + _LAYOUT._serialized_end=523 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/metadata.py b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..fc44459f68fbf59fe2e0fd50fc8bedff4f683367 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/metadata.py @@ -0,0 +1,36 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Information on the custom scalars plugin.""" + + +from tensorboard.compat.proto import summary_pb2 + +# A special tag named used for the summary that stores the layout. +CONFIG_SUMMARY_TAG = "custom_scalars__config__" + +PLUGIN_NAME = "custom_scalars" + + +def create_summary_metadata(): + """Create a `SummaryMetadata` proto for custom scalar plugin data. + + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + return summary_pb2.SummaryMetadata( + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME + ) + ) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/summary.py b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/summary.py new file mode 100644 index 0000000000000000000000000000000000000000..3362233cc7d5000b3b18ae5a19bb4a75624ccebb --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/custom_scalar/summary.py @@ -0,0 +1,80 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Contains summaries related to laying out the custom scalars dashboard.""" + + +from tensorboard.plugins.custom_scalar import layout_pb2 +from tensorboard.plugins.custom_scalar import metadata + + +def op(scalars_layout, collections=None): + """Creates a summary that contains a layout. + + When users navigate to the custom scalars dashboard, they will see a layout + based on the proto provided to this function. + + Args: + scalars_layout: The scalars_layout_pb2.Layout proto that specifies the + layout. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A tensor summary op that writes the layout to disk. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + assert isinstance(scalars_layout, layout_pb2.Layout) + summary_metadata = metadata.create_summary_metadata() + return tf.summary.tensor_summary( + name=metadata.CONFIG_SUMMARY_TAG, + tensor=tf.constant(scalars_layout.SerializeToString(), dtype=tf.string), + collections=collections, + summary_metadata=summary_metadata, + ) + + +def pb(scalars_layout): + """Creates a summary that contains a layout. + + When users navigate to the custom scalars dashboard, they will see a layout + based on the proto provided to this function. + + Args: + scalars_layout: The scalars_layout_pb2.Layout proto that specifies the + layout. + + Returns: + A summary proto containing the layout. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + assert isinstance(scalars_layout, layout_pb2.Layout) + tensor = tf.make_tensor_proto( + scalars_layout.SerializeToString(), dtype=tf.string + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + metadata.create_summary_metadata().SerializeToString() + ) + summary = tf.Summary() + summary.value.add( + tag=metadata.CONFIG_SUMMARY_TAG, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/__init__.py b/lib/python3.12/site-packages/tensorboard/plugins/graph/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2777d9d9ced23eb5d8a34bbb012c988973782bec Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/graph_util.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/graph_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a74800fad23f2bf22862e49ca33155dd53f5c225 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/graph_util.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/graphs_plugin.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/graphs_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bd70170de32e3badae4218861a5eb09286f14102 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/graphs_plugin.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/keras_util.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/keras_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cc5a3a351458c91c32d1118815474b9c1ae53583 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/keras_util.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/metadata.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9ae98ac84a708f57eff3139a3ba686a4a28c3f37 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/graph/__pycache__/metadata.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/graph_util.py b/lib/python3.12/site-packages/tensorboard/plugins/graph/graph_util.py new file mode 100644 index 0000000000000000000000000000000000000000..a3e70a16cab31616f9f0e4d13605a30d881ef3a8 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/graph/graph_util.py @@ -0,0 +1,129 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for graph plugin.""" + +from tensorboard.compat.proto import graph_pb2 + + +def _prefixed_op_name(prefix, op_name): + return "%s/%s" % (prefix, op_name) + + +def _prefixed_func_name(prefix, func_name): + """Returns function name prefixed with `prefix`. + + For function libraries, which are often created out of autographed Python + function, are factored out in the graph vis. They are grouped under a + function name which often has a shape of + `__inference_[py_func_name]_[numeric_suffix]`. + + While it does not have some unique information about which graph it is from, + creating another wrapping structure with graph prefix and "/" is less than + ideal so we join the prefix and func_name using underscore. + + TODO(stephanwlee): add business logic to strip "__inference_" for more user + friendlier name + """ + return "%s_%s" % (prefix, func_name) + + +def _add_with_prepended_names(prefix, graph_to_add, destination_graph): + for node in graph_to_add.node: + new_node = destination_graph.node.add() + new_node.CopyFrom(node) + new_node.name = _prefixed_op_name(prefix, node.name) + new_node.input[:] = [ + _prefixed_op_name(prefix, input_name) for input_name in node.input + ] + + # Remap tf.function method name in the PartitionedCall. 'f' is short for + # function. + if new_node.op == "PartitionedCall" and new_node.attr["f"]: + + new_node.attr["f"].func.name = _prefixed_func_name( + prefix, + new_node.attr["f"].func.name, + ) + + for func in graph_to_add.library.function: + new_func = destination_graph.library.function.add() + new_func.CopyFrom(func) + new_func.signature.name = _prefixed_func_name( + prefix, new_func.signature.name + ) + + for gradient in graph_to_add.library.gradient: + new_gradient = destination_graph.library.gradient.add() + new_gradient.CopyFrom(gradient) + new_gradient.function_name = _prefixed_func_name( + prefix, + new_gradient.function_name, + ) + new_gradient.gradient_func = _prefixed_func_name( + prefix, + new_gradient.gradient_func, + ) + + +def merge_graph_defs(graph_defs): + """Merges GraphDefs by adding unique prefix, `graph_{ind}`, to names. + + All GraphDefs are expected to be of TensorBoard's. + + When collecting graphs using the `tf.summary.trace` API, node names are not + guranteed to be unique. When non-unique names are not considered, it can + lead to graph visualization showing them as one which creates inaccurate + depiction of the flow of the graph (e.g., if there are A -> B -> C and D -> + B -> E, you may see {A, D} -> B -> E). To prevent such graph, we checked + for uniquenss while merging but it resulted in + https://github.com/tensorflow/tensorboard/issues/1929. + + To remedy these issues, we simply "apply name scope" on each graph by + prefixing it with unique name (with a chance of collision) to create + unconnected group of graphs. + + In case there is only one graph def passed, it returns the original + graph_def. In case no graph defs are passed, it returns an empty GraphDef. + + Args: + graph_defs: TensorBoard GraphDefs to merge. + + Returns: + TensorBoard GraphDef that merges all graph_defs with unique prefixes. + + Raises: + ValueError in case GraphDef versions mismatch. + """ + if len(graph_defs) == 1: + return graph_defs[0] + elif len(graph_defs) == 0: + return graph_pb2.GraphDef() + + dst_graph_def = graph_pb2.GraphDef() + + if graph_defs[0].versions.producer: + dst_graph_def.versions.CopyFrom(graph_defs[0].versions) + + for index, graph_def in enumerate(graph_defs): + if dst_graph_def.versions.producer != graph_def.versions.producer: + raise ValueError("Cannot combine GraphDefs of different versions.") + + _add_with_prepended_names( + "graph_%d" % (index + 1), + graph_def, + dst_graph_def, + ) + + return dst_graph_def diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/graphs_plugin.py b/lib/python3.12/site-packages/tensorboard/plugins/graph/graphs_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..9247d1317d7ee0f3e3b542bda4ef1fe480e6a259 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/graph/graphs_plugin.py @@ -0,0 +1,337 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The TensorBoard Graphs plugin.""" + + +import json +from werkzeug import wrappers + +from tensorboard import errors +from tensorboard import plugin_util +from tensorboard.backend import http_util +from tensorboard.backend import process_graph +from tensorboard.compat.proto import config_pb2 +from tensorboard.compat.proto import graph_pb2 +from tensorboard.data import provider +from tensorboard.plugins import base_plugin +from tensorboard.plugins.graph import graph_util +from tensorboard.plugins.graph import keras_util +from tensorboard.plugins.graph import metadata +from tensorboard.util import tb_logging + +logger = tb_logging.get_logger() + + +class GraphsPlugin(base_plugin.TBPlugin): + """Graphs Plugin for TensorBoard.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates GraphsPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._data_provider = context.data_provider + + def get_plugin_apps(self): + return { + "/graph": self.graph_route, + "/info": self.info_route, + "/run_metadata": self.run_metadata_route, + } + + def is_active(self): + """The graphs plugin is active iff any run has a graph or metadata.""" + return False # `list_plugins` as called by TB core suffices + + def data_plugin_names(self): + return ( + metadata.PLUGIN_NAME, + metadata.PLUGIN_NAME_RUN_METADATA, + metadata.PLUGIN_NAME_RUN_METADATA_WITH_GRAPH, + metadata.PLUGIN_NAME_KERAS_MODEL, + metadata.PLUGIN_NAME_TAGGED_RUN_METADATA, + ) + + def frontend_metadata(self): + return base_plugin.FrontendMetadata( + element_name="tf-graph-dashboard", + # TODO(@chihuahua): Reconcile this setting with Health Pills. + disable_reload=True, + ) + + def info_impl(self, ctx, experiment=None): + """Returns a dict of all runs and their data availabilities.""" + result = {} + + def add_row_item(run, tag=None): + run_item = result.setdefault( + run, + { + "run": run, + "tags": {}, + # A run-wide GraphDef of ops. + "run_graph": False, + }, + ) + + tag_item = None + if tag: + tag_item = run_item.get("tags").setdefault( + tag, + { + "tag": tag, + "conceptual_graph": False, + # A tagged GraphDef of ops. + "op_graph": False, + "profile": False, + }, + ) + return (run_item, tag_item) + + mapping = self._data_provider.list_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME_RUN_METADATA_WITH_GRAPH, + ) + for run_name, tags in mapping.items(): + for tag, tag_data in tags.items(): + # The Summary op is defined in TensorFlow and does not use a stringified proto + # as a content of plugin data. It contains single string that denotes a version. + # https://github.com/tensorflow/tensorflow/blob/11f4ecb54708865ec757ca64e4805957b05d7570/tensorflow/python/ops/summary_ops_v2.py#L789-L790 + if tag_data.plugin_content != b"1": + logger.warning( + "Ignoring unrecognizable version of RunMetadata." + ) + continue + (_, tag_item) = add_row_item(run_name, tag) + tag_item["op_graph"] = True + + # Tensors associated with plugin name metadata.PLUGIN_NAME_RUN_METADATA + # contain both op graph and profile information. + mapping = self._data_provider.list_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME_RUN_METADATA, + ) + for run_name, tags in mapping.items(): + for tag, tag_data in tags.items(): + if tag_data.plugin_content != b"1": + logger.warning( + "Ignoring unrecognizable version of RunMetadata." + ) + continue + (_, tag_item) = add_row_item(run_name, tag) + tag_item["profile"] = True + tag_item["op_graph"] = True + + # Tensors associated with plugin name metadata.PLUGIN_NAME_KERAS_MODEL + # contain serialized Keras model in JSON format. + mapping = self._data_provider.list_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME_KERAS_MODEL, + ) + for run_name, tags in mapping.items(): + for tag, tag_data in tags.items(): + if tag_data.plugin_content != b"1": + logger.warning( + "Ignoring unrecognizable version of RunMetadata." + ) + continue + (_, tag_item) = add_row_item(run_name, tag) + tag_item["conceptual_graph"] = True + + mapping = self._data_provider.list_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + ) + for run_name, tags in mapping.items(): + if metadata.RUN_GRAPH_NAME in tags: + (run_item, _) = add_row_item(run_name, None) + run_item["run_graph"] = True + + # Top level `Event.tagged_run_metadata` represents profile data only. + mapping = self._data_provider.list_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME_TAGGED_RUN_METADATA, + ) + for run_name, tags in mapping.items(): + for tag in tags: + (_, tag_item) = add_row_item(run_name, tag) + tag_item["profile"] = True + + return result + + def _read_blob(self, ctx, experiment, plugin_names, run, tag): + for plugin_name in plugin_names: + blob_sequences = self._data_provider.read_blob_sequences( + ctx, + experiment_id=experiment, + plugin_name=plugin_name, + run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), + downsample=1, + ) + blob_sequence_data = blob_sequences.get(run, {}).get(tag, ()) + try: + blob_ref = blob_sequence_data[0].values[0] + except IndexError: + continue + return self._data_provider.read_blob( + ctx, blob_key=blob_ref.blob_key + ) + raise errors.NotFoundError() + + def graph_impl( + self, + ctx, + run, + tag, + is_conceptual, + experiment=None, + limit_attr_size=None, + large_attrs_key=None, + ): + """Result of the form `(body, mime_type)`; may raise `NotFound`.""" + if is_conceptual: + keras_model_config = json.loads( + self._read_blob( + ctx, + experiment, + [metadata.PLUGIN_NAME_KERAS_MODEL], + run, + tag, + ) + ) + graph = keras_util.keras_model_to_graph_def(keras_model_config) + + elif tag is None: + graph_raw = self._read_blob( + ctx, + experiment, + [metadata.PLUGIN_NAME], + run, + metadata.RUN_GRAPH_NAME, + ) + graph = graph_pb2.GraphDef.FromString(graph_raw) + + else: + # Op graph: could be either of two plugins. (Cf. `info_impl`.) + plugins = [ + metadata.PLUGIN_NAME_RUN_METADATA, + metadata.PLUGIN_NAME_RUN_METADATA_WITH_GRAPH, + ] + raw_run_metadata = self._read_blob( + ctx, experiment, plugins, run, tag + ) + run_metadata = config_pb2.RunMetadata.FromString(raw_run_metadata) + graph = graph_util.merge_graph_defs( + [ + func_graph.pre_optimization_graph + for func_graph in run_metadata.function_graphs + ] + ) + + # This next line might raise a ValueError if the limit parameters + # are invalid (size is negative, size present but key absent, etc.). + process_graph.prepare_graph_for_ui( + graph, limit_attr_size, large_attrs_key + ) + return (str(graph), "text/x-protobuf") # pbtxt + + def run_metadata_impl(self, ctx, experiment, run, tag): + """Result of the form `(body, mime_type)`; may raise `NotFound`.""" + # Profile graph: could be either of two plugins. (Cf. `info_impl`.) + plugins = [ + metadata.PLUGIN_NAME_TAGGED_RUN_METADATA, + metadata.PLUGIN_NAME_RUN_METADATA, + ] + raw_run_metadata = self._read_blob(ctx, experiment, plugins, run, tag) + run_metadata = config_pb2.RunMetadata.FromString(raw_run_metadata) + return (str(run_metadata), "text/x-protobuf") # pbtxt + + @wrappers.Request.application + def info_route(self, request): + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + info = self.info_impl(ctx, experiment) + return http_util.Respond(request, info, "application/json") + + @wrappers.Request.application + def graph_route(self, request): + """Given a single run, return the graph definition in protobuf + format.""" + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + run = request.args.get("run") + tag = request.args.get("tag") + conceptual_arg = request.args.get("conceptual", False) + is_conceptual = True if conceptual_arg == "true" else False + + if run is None: + return http_util.Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + + limit_attr_size = request.args.get("limit_attr_size", None) + if limit_attr_size is not None: + try: + limit_attr_size = int(limit_attr_size) + except ValueError: + return http_util.Respond( + request, + "query parameter `limit_attr_size` must be an integer", + "text/plain", + 400, + ) + + large_attrs_key = request.args.get("large_attrs_key", None) + + try: + result = self.graph_impl( + ctx, + run, + tag, + is_conceptual, + experiment, + limit_attr_size, + large_attrs_key, + ) + except ValueError as e: + return http_util.Respond(request, e.message, "text/plain", code=400) + (body, mime_type) = result + return http_util.Respond(request, body, mime_type) + + @wrappers.Request.application + def run_metadata_route(self, request): + """Given a tag and a run, return the session.run() metadata.""" + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + tag = request.args.get("tag") + run = request.args.get("run") + if tag is None: + return http_util.Respond( + request, 'query parameter "tag" is required', "text/plain", 400 + ) + if run is None: + return http_util.Respond( + request, 'query parameter "run" is required', "text/plain", 400 + ) + (body, mime_type) = self.run_metadata_impl(ctx, experiment, run, tag) + return http_util.Respond(request, body, mime_type) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/keras_util.py b/lib/python3.12/site-packages/tensorboard/plugins/graph/keras_util.py new file mode 100644 index 0000000000000000000000000000000000000000..009a48575eb09124f9c852fa5c3c4dd2ad0262a5 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/graph/keras_util.py @@ -0,0 +1,328 @@ +# -*- coding: utf-8 -*- +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for handling Keras model in graph plugin. + +Two canonical types of Keras model are Functional and Sequential. +A model can be serialized as JSON and deserialized to reconstruct a model. +This utility helps with dealing with the serialized Keras model. + +They have distinct structures to the configurations in shapes below: +Functional: + config + name: Name of the model. If not specified, it is 'model' with + an optional suffix if there are more than one instance. + input_layers: Keras.layers.Inputs in the model. + output_layers: Layer names that are outputs of the model. + layers: list of layer configurations. + layer: [*] + inbound_nodes: inputs to this layer. + +Sequential: + config + name: Name of the model. If not specified, it is 'sequential' with + an optional suffix if there are more than one instance. + layers: list of layer configurations. + layer: [*] + +[*]: Note that a model can be a layer. +Please refer to https://github.com/tensorflow/tfjs-layers/blob/master/src/keras_format/model_serialization.ts +for more complete definition. +""" +from tensorboard.compat.proto.graph_pb2 import GraphDef +from tensorboard.compat.tensorflow_stub import dtypes +from tensorboard.util import tb_logging + + +logger = tb_logging.get_logger() + + +def _walk_layers(keras_layer): + """Walks the nested keras layer configuration in preorder. + + Args: + keras_layer: Keras configuration from model.to_json. + + Yields: + A tuple of (name_scope, layer_config). + name_scope: a string representing a scope name, similar to that of tf.name_scope. + layer_config: a dict representing a Keras layer configuration. + """ + yield ("", keras_layer) + if keras_layer.get("config").get("layers"): + name_scope = keras_layer.get("config").get("name") + for layer in keras_layer.get("config").get("layers"): + for sub_name_scope, sublayer in _walk_layers(layer): + sub_name_scope = ( + "%s/%s" % (name_scope, sub_name_scope) + if sub_name_scope + else name_scope + ) + yield (sub_name_scope, sublayer) + + +def _scoped_name(name_scope, node_name): + """Returns scoped name for a node as a string in the form '/'. + + Args: + name_scope: a string representing a scope name, similar to that of tf.name_scope. + node_name: a string representing the current node name. + + Returns + A string representing a scoped name. + """ + if name_scope: + return "%s/%s" % (name_scope, node_name) + return node_name + + +def _is_model(layer): + """Returns True if layer is a model. + + Args: + layer: a dict representing a Keras model configuration. + + Returns: + bool: True if layer is a model. + """ + return layer.get("config").get("layers") is not None + + +def _norm_to_list_of_layers(maybe_layers): + """Normalizes to a list of layers. + + Args: + maybe_layers: A list of data[1] or a list of list of data. + + Returns: + List of list of data. + + [1]: A Functional model has fields 'inbound_nodes' and 'output_layers' which can + look like below: + - ['in_layer_name', 0, 0] + - [['in_layer_is_model', 1, 0], ['in_layer_is_model', 1, 1]] + The data inside the list seems to describe [name, size, index]. + """ + return ( + maybe_layers if isinstance(maybe_layers[0], (list,)) else [maybe_layers] + ) + + +def _get_inbound_nodes(layer): + """Returns a list of [name, size, index] for all inbound nodes of the given layer.""" + inbound_nodes = [] + if layer.get("inbound_nodes") is not None: + for maybe_inbound_node in layer.get("inbound_nodes", []): + if not isinstance(maybe_inbound_node, dict): + # Note that the inbound node parsing is not backward compatible with + # Keras 2. If given a Keras 2 model, the input nodes will be missing + # in the final graph. + continue + for inbound_node_args in maybe_inbound_node.get("args", []): + # Sometimes this field is a list when there are multiple inbound nodes + # for the given layer. + if not isinstance(inbound_node_args, list): + inbound_node_args = [inbound_node_args] + for arg in inbound_node_args: + history = arg.get("config", {}).get("keras_history", []) + if len(history) < 3: + continue + inbound_nodes.append(history[:3]) + return inbound_nodes + + +def _update_dicts( + name_scope, + model_layer, + input_to_in_layer, + model_name_to_output, + prev_node_name, +): + """Updates input_to_in_layer, model_name_to_output, and prev_node_name + based on the model_layer. + + Args: + name_scope: a string representing a scope name, similar to that of tf.name_scope. + model_layer: a dict representing a Keras model configuration. + input_to_in_layer: a dict mapping Keras.layers.Input to inbound layer. + model_name_to_output: a dict mapping Keras Model name to output layer of the model. + prev_node_name: a string representing a previous, in sequential model layout, + node name. + + Returns: + A tuple of (input_to_in_layer, model_name_to_output, prev_node_name). + input_to_in_layer: a dict mapping Keras.layers.Input to inbound layer. + model_name_to_output: a dict mapping Keras Model name to output layer of the model. + prev_node_name: a string representing a previous, in sequential model layout, + node name. + """ + layer_config = model_layer.get("config") + if not layer_config.get("layers"): + raise ValueError("layer is not a model.") + + node_name = _scoped_name(name_scope, layer_config.get("name")) + input_layers = layer_config.get("input_layers") + output_layers = layer_config.get("output_layers") + inbound_nodes = _get_inbound_nodes(model_layer) + + is_functional_model = bool(input_layers and output_layers) + # In case of [1] and the parent model is functional, current layer + # will have the 'inbound_nodes' property. + is_parent_functional_model = bool(inbound_nodes) + + if is_parent_functional_model and is_functional_model: + for input_layer, inbound_node in zip(input_layers, inbound_nodes): + input_layer_name = _scoped_name(node_name, input_layer) + inbound_node_name = _scoped_name(name_scope, inbound_node[0]) + input_to_in_layer[input_layer_name] = inbound_node_name + elif is_parent_functional_model and not is_functional_model: + # Sequential model can take only one input. Make sure inbound to the + # model is linked to the first layer in the Sequential model. + prev_node_name = _scoped_name(name_scope, inbound_nodes[0][0]) + elif ( + not is_parent_functional_model + and prev_node_name + and is_functional_model + ): + assert len(input_layers) == 1, ( + "Cannot have multi-input Functional model when parent model " + "is not Functional. Number of input layers: %d" % len(input_layer) + ) + input_layer = input_layers[0] + input_layer_name = _scoped_name(node_name, input_layer) + input_to_in_layer[input_layer_name] = prev_node_name + + if is_functional_model and output_layers: + layers = _norm_to_list_of_layers(output_layers) + layer_names = [_scoped_name(node_name, layer[0]) for layer in layers] + model_name_to_output[node_name] = layer_names + else: + last_layer = layer_config.get("layers")[-1] + last_layer_name = last_layer.get("config").get("name") + output_node = _scoped_name(node_name, last_layer_name) + model_name_to_output[node_name] = [output_node] + return (input_to_in_layer, model_name_to_output, prev_node_name) + + +def keras_model_to_graph_def(keras_layer): + """Returns a GraphDef representation of the Keras model in a dict form. + + Note that it only supports models that implemented to_json(). + + Args: + keras_layer: A dict from Keras model.to_json(). + + Returns: + A GraphDef representation of the layers in the model. + """ + input_to_layer = {} + model_name_to_output = {} + g = GraphDef() + + # Sequential model layers do not have a field "inbound_nodes" but + # instead are defined implicitly via order of layers. + prev_node_name = None + + for name_scope, layer in _walk_layers(keras_layer): + if _is_model(layer): + ( + input_to_layer, + model_name_to_output, + prev_node_name, + ) = _update_dicts( + name_scope, + layer, + input_to_layer, + model_name_to_output, + prev_node_name, + ) + continue + + layer_config = layer.get("config") + node_name = _scoped_name(name_scope, layer_config.get("name")) + + node_def = g.node.add() + node_def.name = node_name + + if layer.get("class_name") is not None: + keras_cls_name = layer.get("class_name").encode("ascii") + node_def.attr["keras_class"].s = keras_cls_name + + dtype_or_policy = layer_config.get("dtype") + dtype = None + has_unsupported_value = False + # If this is a dict, try and extract the dtype string from + # `config.name`. Keras will export like this for non-input layers and + # some other cases (e.g. tf/keras/mixed_precision/Policy, as described + # in issue #5548). + if isinstance(dtype_or_policy, dict) and "config" in dtype_or_policy: + dtype = dtype_or_policy.get("config").get("name") + elif dtype_or_policy is not None: + dtype = dtype_or_policy + + if dtype is not None: + try: + tf_dtype = dtypes.as_dtype(dtype) + node_def.attr["dtype"].type = tf_dtype.as_datatype_enum + except TypeError: + has_unsupported_value = True + elif dtype_or_policy is not None: + has_unsupported_value = True + + if has_unsupported_value: + # There's at least one known case when this happens, which is when + # mixed precision dtype policies are used, as described in issue + # #5548. (See https://keras.io/api/mixed_precision/). + # There might be a better way to handle this, but here we are. + logger.warning( + "Unsupported dtype value in graph model config (json):\n%s", + dtype_or_policy, + ) + if layer.get("inbound_nodes") is not None: + for name, size, index in _get_inbound_nodes(layer): + inbound_name = _scoped_name(name_scope, name) + # An input to a layer can be output from a model. In that case, the name + # of inbound_nodes to a layer is a name of a model. Remap the name of the + # model to output layer of the model. Also, since there can be multiple + # outputs in a model, make sure we pick the right output_layer from the model. + inbound_node_names = model_name_to_output.get( + inbound_name, [inbound_name] + ) + # There can be multiple inbound_nodes that reference the + # same upstream layer. This causes issues when looking for + # a particular index in that layer, since the indices + # captured in `inbound_nodes` doesn't necessarily match the + # number of entries in the `inbound_node_names` list. To + # avoid IndexErrors, we just use the last element in the + # `inbound_node_names` in this situation. + # Note that this is a quick hack to avoid IndexErrors in + # this situation, and might not be an appropriate solution + # to this problem in general. + input_name = ( + inbound_node_names[index] + if index < len(inbound_node_names) + else inbound_node_names[-1] + ) + node_def.input.append(input_name) + elif prev_node_name is not None: + node_def.input.append(prev_node_name) + + if node_name in input_to_layer: + node_def.input.append(input_to_layer.get(node_name)) + + prev_node_name = node_def.name + + return g diff --git a/lib/python3.12/site-packages/tensorboard/plugins/graph/metadata.py b/lib/python3.12/site-packages/tensorboard/plugins/graph/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..435eefc1226ea419353a4d6697863c9e7669275d --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/graph/metadata.py @@ -0,0 +1,43 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Information on the graph plugin.""" + + +# This name is used as the plugin prefix route and to identify this plugin +# generally, and is also the `plugin_name` for run graphs after data-compat +# transformations. +PLUGIN_NAME = "graphs" +# The Summary API is implemented in TensorFlow because it uses TensorFlow internal APIs. +# As a result, this SummaryMetadata is a bit unconventional and uses non-public +# hardcoded name as the plugin name. Please refer to link below for the summary ops. +# https://github.com/tensorflow/tensorflow/blob/11f4ecb54708865ec757ca64e4805957b05d7570/tensorflow/python/ops/summary_ops_v2.py#L757 +PLUGIN_NAME_RUN_METADATA = "graph_run_metadata" +# https://github.com/tensorflow/tensorflow/blob/11f4ecb54708865ec757ca64e4805957b05d7570/tensorflow/python/ops/summary_ops_v2.py#L788 +PLUGIN_NAME_RUN_METADATA_WITH_GRAPH = "graph_run_metadata_graph" +# https://github.com/tensorflow/tensorflow/blob/565952cc2f17fdfd995e25171cf07be0f6f06180/tensorflow/python/ops/summary_ops_v2.py#L825 +PLUGIN_NAME_KERAS_MODEL = "graph_keras_model" +# Plugin name used for `Event.tagged_run_metadata`. This doesn't fall into one +# of the above cases because (despite the name) `PLUGIN_NAME_RUN_METADATA` is +# _required_ to have both profile and op graphs, whereas tagged run metadata +# need only have profile data. +PLUGIN_NAME_TAGGED_RUN_METADATA = "graph_tagged_run_metadata" + +# In the context of the data provider interface, tag name given to a +# graph read from the `graph_def` field of an `Event` proto, which is +# not attached to a summary and thus does not have a proper tag name of +# its own. Run level graphs always represent `GraphDef`s (graphs of +# TensorFlow ops), never conceptual graphs, profile graphs, etc. This is +# the only tag name used by the `"graphs"` plugin. +RUN_GRAPH_NAME = "__run_graph__" diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/__init__.py b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..515e9555627bf23f703ce4886e7b814d8320f57d Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/histograms_plugin.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/histograms_plugin.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..333ade1f5315b91e3d1c41508c3e4f467e6e205a Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/histograms_plugin.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/metadata.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b84dadcd48ee15ec3c5ffa30aa743f81c91a141c Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/metadata.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/plugin_data_pb2.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/plugin_data_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7256c6530daf7273dd7113e4674acf73624bf09 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/plugin_data_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/summary.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/summary.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..57688dc2d7456ab7bbe69505106369a11f533457 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/summary.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/summary_v2.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/summary_v2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f4e6bb5ffc7b08ec2431e1fc16b6c85a4dc22819 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/plugins/histogram/__pycache__/summary_v2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/histograms_plugin.py b/lib/python3.12/site-packages/tensorboard/plugins/histogram/histograms_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..cace2d12764d36b16e99415eb6ad84a866726bc5 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/histogram/histograms_plugin.py @@ -0,0 +1,146 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""The TensorBoard Histograms plugin. + +See `http_api.md` in this directory for specifications of the routes for +this plugin. +""" + + +from werkzeug import wrappers + +from tensorboard import errors +from tensorboard import plugin_util +from tensorboard.backend import http_util +from tensorboard.data import provider +from tensorboard.plugins import base_plugin +from tensorboard.plugins.histogram import metadata + + +_DEFAULT_DOWNSAMPLING = 500 # histograms per time series + + +class HistogramsPlugin(base_plugin.TBPlugin): + """Histograms Plugin for TensorBoard. + + This supports both old-style summaries (created with TensorFlow ops + that output directly to the `histo` field of the proto) and new- + style summaries (as created by the + `tensorboard.plugins.histogram.summary` module). + """ + + plugin_name = metadata.PLUGIN_NAME + + # Use a round number + 1 since sampling includes both start and end steps, + # so N+1 samples corresponds to dividing the step sequence into N intervals. + SAMPLE_SIZE = 51 + + def __init__(self, context): + """Instantiates HistogramsPlugin via TensorBoard core. + + Args: + context: A base_plugin.TBContext instance. + """ + self._downsample_to = (context.sampling_hints or {}).get( + self.plugin_name, _DEFAULT_DOWNSAMPLING + ) + self._data_provider = context.data_provider + self._version_checker = plugin_util._MetadataVersionChecker( + data_kind="histogram", + latest_known_version=0, + ) + + def get_plugin_apps(self): + return { + "/histograms": self.histograms_route, + "/tags": self.tags_route, + } + + def is_active(self): + return False # `list_plugins` as called by TB core suffices + + def index_impl(self, ctx, experiment): + """Return {runName: {tagName: {displayName: ..., description: + ...}}}.""" + mapping = self._data_provider.list_tensors( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + ) + result = {run: {} for run in mapping} + for run, tag_to_content in mapping.items(): + for tag, metadatum in tag_to_content.items(): + description = plugin_util.markdown_to_safe_html( + metadatum.description + ) + md = metadata.parse_plugin_metadata(metadatum.plugin_content) + if not self._version_checker.ok(md.version, run, tag): + continue + result[run][tag] = { + "displayName": metadatum.display_name, + "description": description, + } + return result + + def frontend_metadata(self): + return base_plugin.FrontendMetadata( + element_name="tf-histogram-dashboard" + ) + + def histograms_impl(self, ctx, tag, run, experiment, downsample_to=None): + """Result of the form `(body, mime_type)`. + + At most `downsample_to` events will be returned. If this value is + `None`, then default downsampling will be performed. + + Raises: + tensorboard.errors.PublicError: On invalid request. + """ + sample_count = ( + downsample_to if downsample_to is not None else self._downsample_to + ) + all_histograms = self._data_provider.read_tensors( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + downsample=sample_count, + run_tag_filter=provider.RunTagFilter(runs=[run], tags=[tag]), + ) + histograms = all_histograms.get(run, {}).get(tag, None) + if histograms is None: + raise errors.NotFoundError( + "No histogram tag %r for run %r" % (tag, run) + ) + events = [(e.wall_time, e.step, e.numpy.tolist()) for e in histograms] + return (events, "application/json") + + @wrappers.Request.application + def tags_route(self, request): + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + index = self.index_impl(ctx, experiment=experiment) + return http_util.Respond(request, index, "application/json") + + @wrappers.Request.application + def histograms_route(self, request): + """Given a tag and single run, return array of histogram values.""" + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + tag = request.args.get("tag") + run = request.args.get("run") + (body, mime_type) = self.histograms_impl( + ctx, tag, run, experiment=experiment, downsample_to=self.SAMPLE_SIZE + ) + return http_util.Respond(request, body, mime_type) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/metadata.py b/lib/python3.12/site-packages/tensorboard/plugins/histogram/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..0cdbfeeac18623f9a709d60c51fe4ee996a650bc --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/histogram/metadata.py @@ -0,0 +1,64 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Information about histogram summaries.""" + + +from tensorboard.compat.proto import summary_pb2 +from tensorboard.plugins.histogram import plugin_data_pb2 + +PLUGIN_NAME = "histograms" + +# The most recent value for the `version` field of the +# `HistogramPluginData` proto. +PROTO_VERSION = 0 + + +def create_summary_metadata(display_name, description): + """Create a `summary_pb2.SummaryMetadata` proto for histogram plugin data. + + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + content = plugin_data_pb2.HistogramPluginData(version=PROTO_VERSION) + return summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content.SerializeToString() + ), + ) + + +def parse_plugin_metadata(content): + """Parse summary metadata to a Python object. + + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the histogram plugin. + + Returns: + A `HistogramPluginData` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + if content == b"{}": + # Old-style JSON format. Equivalent to an all-default proto. + return plugin_data_pb2.HistogramPluginData() + else: + result = plugin_data_pb2.HistogramPluginData.FromString(content) + if result.version == 0: + return result + # No other versions known at this time, so no migrations to do. + return result diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/plugin_data_pb2.py b/lib/python3.12/site-packages/tensorboard/plugins/histogram/plugin_data_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..309eb16c5d8aefcc0b93f57abc717c854f328f92 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/histogram/plugin_data_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorboard/plugins/histogram/plugin_data.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n/tensorboard/plugins/histogram/plugin_data.proto\x12\x0btensorboard\"&\n\x13HistogramPluginData\x12\x0f\n\x07version\x18\x01 \x01(\x05\x62\x06proto3') + + + +_HISTOGRAMPLUGINDATA = DESCRIPTOR.message_types_by_name['HistogramPluginData'] +HistogramPluginData = _reflection.GeneratedProtocolMessageType('HistogramPluginData', (_message.Message,), { + 'DESCRIPTOR' : _HISTOGRAMPLUGINDATA, + '__module__' : 'tensorboard.plugins.histogram.plugin_data_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.HistogramPluginData) + }) +_sym_db.RegisterMessage(HistogramPluginData) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _HISTOGRAMPLUGINDATA._serialized_start=64 + _HISTOGRAMPLUGINDATA._serialized_end=102 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/summary.py b/lib/python3.12/site-packages/tensorboard/plugins/histogram/summary.py new file mode 100644 index 0000000000000000000000000000000000000000..d804852f63843f25cfb99d71b1d3d6adb86659b6 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/histogram/summary.py @@ -0,0 +1,236 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Histogram summaries and TensorFlow operations to create them. + +A histogram summary stores a list of buckets. Each bucket is encoded as +a triple `[left_edge, right_edge, count]`. Thus, a full histogram is +encoded as a tensor of dimension `[k, 3]`. + +In general, the value of `k` (the number of buckets) will be a constant, +like 30. There are two edge cases: if there is no data, then there are +no buckets (the shape is `[0, 3]`); and if there is data but all points +have the same value, then there is one bucket whose left and right +endpoints are the same (the shape is `[1, 3]`). + +NOTE: This module is in beta, and its API is subject to change, but the +data that it stores to disk will be supported forever. +""" + + +import numpy as np + +from tensorboard.plugins.histogram import metadata +from tensorboard.plugins.histogram import summary_v2 + + +# Export V3 versions. +histogram = summary_v2.histogram +histogram_pb = summary_v2.histogram_pb + + +def _buckets(data, bucket_count=None): + """Create a TensorFlow op to group data into histogram buckets. + + Arguments: + data: A `Tensor` of any shape. Must be castable to `float64`. + bucket_count: Optional positive `int` or scalar `int32` `Tensor`. + Returns: + A `Tensor` of shape `[k, 3]` and type `float64`. The `i`th row is + a triple `[left_edge, right_edge, count]` for a single bucket. + The value of `k` is either `bucket_count` or `1` or `0`. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if bucket_count is None: + bucket_count = summary_v2.DEFAULT_BUCKET_COUNT + with tf.name_scope( + "buckets", values=[data, bucket_count] + ), tf.control_dependencies( + [tf.assert_scalar(bucket_count), tf.assert_type(bucket_count, tf.int32)] + ): + data = tf.reshape(data, shape=[-1]) # flatten + data = tf.cast(data, tf.float64) + is_empty = tf.equal(tf.size(input=data), 0) + + def when_empty(): + return tf.constant([], shape=(0, 3), dtype=tf.float64) + + def when_nonempty(): + min_ = tf.reduce_min(input_tensor=data) + max_ = tf.reduce_max(input_tensor=data) + range_ = max_ - min_ + is_singular = tf.equal(range_, 0) + + def when_nonsingular(): + bucket_width = range_ / tf.cast(bucket_count, tf.float64) + offsets = data - min_ + bucket_indices = tf.cast( + tf.floor(offsets / bucket_width), dtype=tf.int32 + ) + clamped_indices = tf.minimum(bucket_indices, bucket_count - 1) + # Use float64 instead of float32 to avoid accumulating floating point error + # later in tf.reduce_sum when summing more than 2^24 individual `1.0` values. + # See https://github.com/tensorflow/tensorflow/issues/51419 for details. + one_hots = tf.one_hot( + clamped_indices, depth=bucket_count, dtype=tf.float64 + ) + bucket_counts = tf.cast( + tf.reduce_sum(input_tensor=one_hots, axis=0), + dtype=tf.float64, + ) + edges = tf.linspace(min_, max_, bucket_count + 1) + left_edges = edges[:-1] + right_edges = edges[1:] + return tf.transpose( + a=tf.stack([left_edges, right_edges, bucket_counts]) + ) + + def when_singular(): + center = min_ + bucket_starts = tf.stack([center - 0.5]) + bucket_ends = tf.stack([center + 0.5]) + bucket_counts = tf.stack( + [tf.cast(tf.size(input=data), tf.float64)] + ) + return tf.transpose( + a=tf.stack([bucket_starts, bucket_ends, bucket_counts]) + ) + + return tf.cond(is_singular, when_singular, when_nonsingular) + + return tf.cond(is_empty, when_empty, when_nonempty) + + +def op( + name, + data, + bucket_count=None, + display_name=None, + description=None, + collections=None, +): + """Create a legacy histogram summary op. + + Arguments: + name: A unique name for the generated summary node. + data: A `Tensor` of any shape. Must be castable to `float64`. + bucket_count: Optional positive `int`. The output will have this + many buckets, except in two edge cases. If there is no data, then + there are no buckets. If there is data but all points have the + same value, then there is one bucket whose left and right + endpoints are the same. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A TensorFlow summary op. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + with tf.name_scope(name): + tensor = _buckets(data, bucket_count=bucket_count) + return tf.summary.tensor_summary( + name="histogram_summary", + tensor=tensor, + collections=collections, + summary_metadata=summary_metadata, + ) + + +def pb(name, data, bucket_count=None, display_name=None, description=None): + """Create a legacy histogram summary protobuf. + + Arguments: + name: A unique name for the generated summary, including any desired + name scopes. + data: A `np.array` or array-like form of any shape. Must have type + castable to `float`. + bucket_count: Optional positive `int`. The output will have this + many buckets, except in two edge cases. If there is no data, then + there are no buckets. If there is data but all points have the + same value, then there is one bucket whose left and right + endpoints are the same. + display_name: Optional name for this summary in TensorBoard, as a + `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `tf.Summary` protobuf object. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if bucket_count is None: + bucket_count = summary_v2.DEFAULT_BUCKET_COUNT + data = np.array(data).flatten().astype(float) + if data.size == 0: + buckets = np.array([]).reshape((0, 3)) + else: + min_ = np.min(data) + max_ = np.max(data) + range_ = max_ - min_ + if range_ == 0: + center = min_ + buckets = np.array([[center - 0.5, center + 0.5, float(data.size)]]) + else: + bucket_width = range_ / bucket_count + offsets = data - min_ + bucket_indices = np.floor(offsets / bucket_width).astype(int) + clamped_indices = np.minimum(bucket_indices, bucket_count - 1) + one_hots = np.array([clamped_indices]).transpose() == np.arange( + 0, bucket_count + ) # broadcast + assert one_hots.shape == (data.size, bucket_count), ( + one_hots.shape, + (data.size, bucket_count), + ) + bucket_counts = np.sum(one_hots, axis=0) + edges = np.linspace(min_, max_, bucket_count + 1) + left_edges = edges[:-1] + right_edges = edges[1:] + buckets = np.array( + [left_edges, right_edges, bucket_counts] + ).transpose() + tensor = tf.make_tensor_proto(buckets, dtype=tf.float64) + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name, description=description + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + + summary = tf.Summary() + summary.value.add( + tag="%s/histogram_summary" % name, + metadata=tf_summary_metadata, + tensor=tensor, + ) + return summary diff --git a/lib/python3.12/site-packages/tensorboard/plugins/histogram/summary_v2.py b/lib/python3.12/site-packages/tensorboard/plugins/histogram/summary_v2.py new file mode 100644 index 0000000000000000000000000000000000000000..d64a9e45598f72a17f2402b3c897d62850074c1b --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/histogram/summary_v2.py @@ -0,0 +1,293 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Histogram summaries and TensorFlow operations to create them, V2 versions. + +A histogram summary stores a list of buckets. Each bucket is encoded as a triple +`[left_edge, right_edge, count]`. Thus, a full histogram is encoded as a tensor +of dimension `[k, 3]`, where the first `k - 1` buckets are closed-open and the +last bucket is closed-closed. + +In general, the shape of the output histogram is always constant (`[k, 3]`). +In the case of empty data, the output will be an all-zero histogram of shape +`[k, 3]`, where all edges and counts are zeros. If there is data but all points +have the same value, then all buckets' left and right edges are the same and only +the last bucket has nonzero count. +""" + +import numpy as np + +from tensorboard.compat import tf2 as tf +from tensorboard.compat.proto import summary_pb2 +from tensorboard.plugins.histogram import metadata +from tensorboard.util import lazy_tensor_creator +from tensorboard.util import tensor_util + + +DEFAULT_BUCKET_COUNT = 30 + + +def histogram_pb(tag, data, buckets=None, description=None): + """Create a histogram summary protobuf. + + Arguments: + tag: String tag for the summary. + data: A `np.array` or array-like form of any shape. Must have type + castable to `float`. + buckets: Optional positive `int`. The output shape will always be + [buckets, 3]. If there is no data, then an all-zero array of shape + [buckets, 3] will be returned. If there is data but all points have + the same value, then all buckets' left and right endpoints are the + same and only the last bucket has nonzero count. Defaults to 30 if + not specified. + description: Optional long-form description for this summary, as a + `str`. Markdown is supported. Defaults to empty. + + Returns: + A `summary_pb2.Summary` protobuf object. + """ + bucket_count = DEFAULT_BUCKET_COUNT if buckets is None else buckets + data = np.array(data).flatten().astype(float) + if bucket_count == 0 or data.size == 0: + histogram_buckets = np.zeros((bucket_count, 3)) + else: + min_ = np.min(data) + max_ = np.max(data) + range_ = max_ - min_ + if range_ == 0: + left_edges = right_edges = np.array([min_] * bucket_count) + bucket_counts = np.array([0] * (bucket_count - 1) + [data.size]) + histogram_buckets = np.array( + [left_edges, right_edges, bucket_counts] + ).transpose() + else: + bucket_width = range_ / bucket_count + offsets = data - min_ + bucket_indices = np.floor(offsets / bucket_width).astype(int) + clamped_indices = np.minimum(bucket_indices, bucket_count - 1) + one_hots = np.array([clamped_indices]).transpose() == np.arange( + 0, bucket_count + ) # broadcast + assert one_hots.shape == (data.size, bucket_count), ( + one_hots.shape, + (data.size, bucket_count), + ) + bucket_counts = np.sum(one_hots, axis=0) + edges = np.linspace(min_, max_, bucket_count + 1) + left_edges = edges[:-1] + right_edges = edges[1:] + histogram_buckets = np.array( + [left_edges, right_edges, bucket_counts] + ).transpose() + tensor = tensor_util.make_tensor_proto(histogram_buckets, dtype=np.float64) + + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + summary = summary_pb2.Summary() + summary.value.add(tag=tag, metadata=summary_metadata, tensor=tensor) + return summary + + +# This is the TPU compatible V3 histogram implementation as of 2021-12-01. +def histogram(name, data, step=None, buckets=None, description=None): + """Write a histogram summary. + + See also `tf.summary.scalar`, `tf.summary.SummaryWriter`. + + Writes a histogram to the current default summary writer, for later analysis + in TensorBoard's 'Histograms' and 'Distributions' dashboards (data written + using this API will appear in both places). Like `tf.summary.scalar` points, + each histogram is associated with a `step` and a `name`. All the histograms + with the same `name` constitute a time series of histograms. + + The histogram is calculated over all the elements of the given `Tensor` + without regard to its shape or rank. + + This example writes 2 histograms: + + ```python + w = tf.summary.create_file_writer('test/logs') + with w.as_default(): + tf.summary.histogram("activations", tf.random.uniform([100, 50]), step=0) + tf.summary.histogram("initial_weights", tf.random.normal([1000]), step=0) + ``` + + A common use case is to examine the changing activation patterns (or lack + thereof) at specific layers in a neural network, over time. + + ```python + w = tf.summary.create_file_writer('test/logs') + with w.as_default(): + for step in range(100): + # Generate fake "activations". + activations = [ + tf.random.normal([1000], mean=step, stddev=1), + tf.random.normal([1000], mean=step, stddev=10), + tf.random.normal([1000], mean=step, stddev=100), + ] + + tf.summary.histogram("layer1/activate", activations[0], step=step) + tf.summary.histogram("layer2/activate", activations[1], step=step) + tf.summary.histogram("layer3/activate", activations[2], step=step) + ``` + + Arguments: + name: A name for this summary. The summary tag used for TensorBoard will + be this name prefixed by any active name scopes. + data: A `Tensor` of any shape. The histogram is computed over its elements, + which must be castable to `float64`. + step: Explicit `int64`-castable monotonic step value for this summary. If + omitted, this defaults to `tf.summary.experimental.get_step()`, which must + not be None. + buckets: Optional positive `int`. The output will have this + many buckets, except in two edge cases. If there is no data, then + there are no buckets. If there is data but all points have the + same value, then all buckets' left and right endpoints are the same + and only the last bucket has nonzero count. Defaults to 30 if not + specified. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + + Returns: + True on success, or false if no summary was emitted because no default + summary writer was available. + + Raises: + ValueError: if a default writer exists, but no step was provided and + `tf.summary.experimental.get_step()` is None. + """ + # Avoid building unused gradient graphs for conds below. This works around + # an error building second-order gradient graphs when XlaDynamicUpdateSlice + # is used, and will generally speed up graph building slightly. + data = tf.stop_gradient(data) + summary_metadata = metadata.create_summary_metadata( + display_name=None, description=description + ) + # TODO(https://github.com/tensorflow/tensorboard/issues/2109): remove fallback + summary_scope = ( + getattr(tf.summary.experimental, "summary_scope", None) + or tf.summary.summary_scope + ) + + # TODO(ytjing): add special case handling. + with summary_scope( + name, "histogram_summary", values=[data, buckets, step] + ) as (tag, _): + # Defer histogram bucketing logic by passing it as a callable to + # write(), wrapped in a LazyTensorCreator for backwards + # compatibility, so that we only do this work when summaries are + # actually written. + @lazy_tensor_creator.LazyTensorCreator + def lazy_tensor(): + return _buckets(data, buckets) + + return tf.summary.write( + tag=tag, + tensor=lazy_tensor, + step=step, + metadata=summary_metadata, + ) + + +def _buckets(data, bucket_count=None): + """Create a TensorFlow op to group data into histogram buckets. + + Arguments: + data: A `Tensor` of any shape. Must be castable to `float64`. + bucket_count: Optional non-negative `int` or scalar `int32` `Tensor`, + defaults to 30. + Returns: + A `Tensor` of shape `[k, 3]` and type `float64`. The `i`th row is + a triple `[left_edge, right_edge, count]` for a single bucket. + The value of `k` is either `bucket_count` or `0` (when input data + is empty). + """ + if bucket_count is None: + bucket_count = DEFAULT_BUCKET_COUNT + with tf.name_scope("buckets"): + tf.debugging.assert_scalar(bucket_count) + tf.debugging.assert_type(bucket_count, tf.int32) + # Treat a negative bucket count as zero. + bucket_count = tf.math.maximum(0, bucket_count) + data = tf.reshape(data, shape=[-1]) # flatten + data = tf.cast(data, tf.float64) + data_size = tf.size(input=data) + is_empty = tf.logical_or( + tf.equal(data_size, 0), tf.less_equal(bucket_count, 0) + ) + + def when_empty(): + """When input data is empty or bucket_count is zero. + + 1. If bucket_count is specified as zero, an empty tensor of shape + (0, 3) will be returned. + 2. If the input data is empty, a tensor of shape (bucket_count, 3) + of all zero values will be returned. + """ + return tf.zeros((bucket_count, 3), dtype=tf.float64) + + def when_nonempty(): + min_ = tf.reduce_min(input_tensor=data) + max_ = tf.reduce_max(input_tensor=data) + range_ = max_ - min_ + has_single_value = tf.equal(range_, 0) + + def when_multiple_values(): + """When input data contains multiple values.""" + bucket_width = range_ / tf.cast(bucket_count, tf.float64) + offsets = data - min_ + bucket_indices = tf.cast( + tf.floor(offsets / bucket_width), dtype=tf.int32 + ) + clamped_indices = tf.minimum(bucket_indices, bucket_count - 1) + # Use float64 instead of float32 to avoid accumulating floating point error + # later in tf.reduce_sum when summing more than 2^24 individual `1.0` values. + # See https://github.com/tensorflow/tensorflow/issues/51419 for details. + one_hots = tf.one_hot( + clamped_indices, depth=bucket_count, dtype=tf.float64 + ) + bucket_counts = tf.cast( + tf.reduce_sum(input_tensor=one_hots, axis=0), + dtype=tf.float64, + ) + edges = tf.linspace(min_, max_, bucket_count + 1) + # Ensure edges[-1] == max_, which TF's linspace implementation does not + # do, leaving it subject to the whim of floating point rounding error. + edges = tf.concat([edges[:-1], [max_]], 0) + left_edges = edges[:-1] + right_edges = edges[1:] + return tf.transpose( + a=tf.stack([left_edges, right_edges, bucket_counts]) + ) + + def when_single_value(): + """When input data contains a single unique value.""" + # Left and right edges are the same for single value input. + edges = tf.fill([bucket_count], max_) + # Bucket counts are 0 except the last bucket (if bucket_count > 0), + # which is `data_size`. Ensure that the resulting counts vector has + # length `bucket_count` always, including the bucket_count==0 case. + zeroes = tf.fill([bucket_count], 0) + bucket_counts = tf.cast( + tf.concat([zeroes[:-1], [data_size]], 0)[:bucket_count], + dtype=tf.float64, + ) + return tf.transpose(a=tf.stack([edges, edges, bucket_counts])) + + return tf.cond( + has_single_value, when_single_value, when_multiple_values + ) + + return tf.cond(is_empty, when_empty, when_nonempty) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/__init__.py b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/metadata.py b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..d94dc15752852786ed8ab73d6c3e8fa77669d5d3 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/metadata.py @@ -0,0 +1,76 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Internal information about the pr_curves plugin.""" + + +from tensorboard.compat.proto import summary_pb2 +from tensorboard.plugins.pr_curve import plugin_data_pb2 + +PLUGIN_NAME = "pr_curves" + +# Indices for obtaining various values from the tensor stored in a summary. +TRUE_POSITIVES_INDEX = 0 +FALSE_POSITIVES_INDEX = 1 +TRUE_NEGATIVES_INDEX = 2 +FALSE_NEGATIVES_INDEX = 3 +PRECISION_INDEX = 4 +RECALL_INDEX = 5 + +# The most recent value for the `version` field of the +# `PrCurvePluginData` proto. +PROTO_VERSION = 0 + + +def create_summary_metadata(display_name, description, num_thresholds): + """Create a `summary_pb2.SummaryMetadata` proto for pr_curves plugin data. + + Arguments: + display_name: The display name used in TensorBoard. + description: The description to show in TensorBoard. + num_thresholds: The number of thresholds to use for PR curves. + + Returns: + A `summary_pb2.SummaryMetadata` protobuf object. + """ + pr_curve_plugin_data = plugin_data_pb2.PrCurvePluginData( + version=PROTO_VERSION, num_thresholds=num_thresholds + ) + content = pr_curve_plugin_data.SerializeToString() + return summary_pb2.SummaryMetadata( + display_name=display_name, + summary_description=description, + plugin_data=summary_pb2.SummaryMetadata.PluginData( + plugin_name=PLUGIN_NAME, content=content + ), + ) + + +def parse_plugin_metadata(content): + """Parse summary metadata to a Python object. + + Arguments: + content: The `content` field of a `SummaryMetadata` proto + corresponding to the pr_curves plugin. + + Returns: + A `PrCurvesPlugin` protobuf object. + """ + if not isinstance(content, bytes): + raise TypeError("Content type must be bytes") + result = plugin_data_pb2.PrCurvePluginData.FromString(content) + if result.version == 0: + return result + # No other versions known at this time, so no migrations to do. + return result diff --git a/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/plugin_data_pb2.py b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/plugin_data_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..aa0659dda3c358dbba73d25d7fddad1a55fbd0a0 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/plugin_data_pb2.py @@ -0,0 +1,34 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorboard/plugins/pr_curve/plugin_data.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n.tensorboard/plugins/pr_curve/plugin_data.proto\x12\x0btensorboard\"<\n\x11PrCurvePluginData\x12\x0f\n\x07version\x18\x01 \x01(\x05\x12\x16\n\x0enum_thresholds\x18\x02 \x01(\rb\x06proto3') + + + +_PRCURVEPLUGINDATA = DESCRIPTOR.message_types_by_name['PrCurvePluginData'] +PrCurvePluginData = _reflection.GeneratedProtocolMessageType('PrCurvePluginData', (_message.Message,), { + 'DESCRIPTOR' : _PRCURVEPLUGINDATA, + '__module__' : 'tensorboard.plugins.pr_curve.plugin_data_pb2' + # @@protoc_insertion_point(class_scope:tensorboard.PrCurvePluginData) + }) +_sym_db.RegisterMessage(PrCurvePluginData) + +if _descriptor._USE_C_DESCRIPTORS == False: + + DESCRIPTOR._options = None + _PRCURVEPLUGINDATA._serialized_start=63 + _PRCURVEPLUGINDATA._serialized_end=123 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/pr_curves_plugin.py b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/pr_curves_plugin.py new file mode 100644 index 0000000000000000000000000000000000000000..92837d5bdf742f55cd0f1f216310f45e9205bc7c --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/pr_curves_plugin.py @@ -0,0 +1,242 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import numpy as np +from werkzeug import wrappers + +from tensorboard import plugin_util +from tensorboard.data import provider +from tensorboard.backend import http_util +from tensorboard.plugins import base_plugin +from tensorboard.plugins.pr_curve import metadata + + +_DEFAULT_DOWNSAMPLING = 100 # PR curves per time series + + +class PrCurvesPlugin(base_plugin.TBPlugin): + """A plugin that serves PR curves for individual classes.""" + + plugin_name = metadata.PLUGIN_NAME + + def __init__(self, context): + """Instantiates a PrCurvesPlugin. + + Args: + context: A base_plugin.TBContext instance. A magic container that + TensorBoard uses to make objects available to the plugin. + """ + self._data_provider = context.data_provider + self._downsample_to = (context.sampling_hints or {}).get( + metadata.PLUGIN_NAME, _DEFAULT_DOWNSAMPLING + ) + self._version_checker = plugin_util._MetadataVersionChecker( + data_kind="PR curve", + latest_known_version=0, + ) + + @wrappers.Request.application + def pr_curves_route(self, request): + """A route that returns a JSON mapping between runs and PR curve data. + + Returns: + Given a tag and a comma-separated list of runs (both stored within GET + parameters), fetches a JSON object that maps between run name and objects + containing data required for PR curves for that run. Runs that either + cannot be found or that lack tags will be excluded from the response. + """ + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + + runs = request.args.getlist("run") + if not runs: + return http_util.Respond( + request, "No runs provided when fetching PR curve data", 400 + ) + + tag = request.args.get("tag") + if not tag: + return http_util.Respond( + request, "No tag provided when fetching PR curve data", 400 + ) + + try: + response = http_util.Respond( + request, + self.pr_curves_impl(ctx, experiment, runs, tag), + "application/json", + ) + except ValueError as e: + return http_util.Respond(request, str(e), "text/plain", 400) + + return response + + def pr_curves_impl(self, ctx, experiment, runs, tag): + """Creates the JSON object for the PR curves response for a run-tag + combo. + + Arguments: + runs: A list of runs to fetch the curves for. + tag: The tag to fetch the curves for. + + Raises: + ValueError: If no PR curves could be fetched for a run and tag. + + Returns: + The JSON object for the PR curves route response. + """ + response_mapping = {} + rtf = provider.RunTagFilter(runs, [tag]) + read_result = self._data_provider.read_tensors( + ctx, + experiment_id=experiment, + plugin_name=metadata.PLUGIN_NAME, + run_tag_filter=rtf, + downsample=self._downsample_to, + ) + for run in runs: + data = read_result.get(run, {}).get(tag) + if data is None: + raise ValueError( + "No PR curves could be found for run %r and tag %r" + % (run, tag) + ) + response_mapping[run] = [self._process_datum(d) for d in data] + return response_mapping + + @wrappers.Request.application + def tags_route(self, request): + """A route (HTTP handler) that returns a response with tags. + + Returns: + A response that contains a JSON object. The keys of the object + are all the runs. Each run is mapped to a (potentially empty) dictionary + whose keys are tags associated with run and whose values are metadata + (dictionaries). + + The metadata dictionaries contain 2 keys: + - displayName: For the display name used atop visualizations in + TensorBoard. + - description: The description that appears near visualizations upon the + user hovering over a certain icon. + """ + ctx = plugin_util.context(request.environ) + experiment = plugin_util.experiment_id(request.environ) + return http_util.Respond( + request, self.tags_impl(ctx, experiment), "application/json" + ) + + def tags_impl(self, ctx, experiment): + """Creates the JSON object for the tags route response. + + Returns: + The JSON object for the tags route response. + """ + mapping = self._data_provider.list_tensors( + ctx, experiment_id=experiment, plugin_name=metadata.PLUGIN_NAME + ) + result = {run: {} for run in mapping} + for run, tag_to_time_series in mapping.items(): + for tag, time_series in tag_to_time_series.items(): + md = metadata.parse_plugin_metadata(time_series.plugin_content) + if not self._version_checker.ok(md.version, run, tag): + continue + result[run][tag] = { + "displayName": time_series.display_name, + "description": plugin_util.markdown_to_safe_html( + time_series.description + ), + } + return result + + def get_plugin_apps(self): + """Gets all routes offered by the plugin. + + Returns: + A dictionary mapping URL path to route that handles it. + """ + return { + "/tags": self.tags_route, + "/pr_curves": self.pr_curves_route, + } + + def is_active(self): + return False # `list_plugins` as called by TB core suffices + + def frontend_metadata(self): + return base_plugin.FrontendMetadata( + element_name="tf-pr-curve-dashboard", + tab_name="PR Curves", + ) + + def _process_datum(self, datum): + """Converts a TensorDatum into a dict that encapsulates information on + it. + + Args: + datum: The TensorDatum to convert. + + Returns: + A JSON-able dictionary of PR curve data for 1 step. + """ + return self._make_pr_entry(datum.step, datum.wall_time, datum.numpy) + + def _make_pr_entry(self, step, wall_time, data_array): + """Creates an entry for PR curve data. Each entry corresponds to 1 + step. + + Args: + step: The step. + wall_time: The wall time. + data_array: A numpy array of PR curve data stored in the summary format. + + Returns: + A PR curve entry. + """ + tp_index = metadata.TRUE_POSITIVES_INDEX + fp_index = metadata.FALSE_POSITIVES_INDEX + tn_index = metadata.TRUE_NEGATIVES_INDEX + fn_index = metadata.FALSE_NEGATIVES_INDEX + + # Trim entries for which TP + FP = 0 (precision is undefined) at the tail of + # the data. + positives = data_array[[tp_index, fp_index], :].astype(int).sum(axis=0) + # Searching from the end, find the farthest index where TP + FP = 0. + end_index_inclusive = len(positives) - 1 + while end_index_inclusive > 0 and positives[end_index_inclusive] == 0: + end_index_inclusive -= 1 + end_index = end_index_inclusive + 1 + # Generate thresholds in [0, 1]. + num_thresholds = data_array.shape[1] + thresholds = np.linspace(0.0, 1.0, num_thresholds) + + true_positives = [int(v) for v in data_array[tp_index]] + false_positives = [int(v) for v in data_array[fp_index]] + true_negatives = [int(v) for v in data_array[tn_index]] + false_negatives = [int(v) for v in data_array[fn_index]] + + return { + "wall_time": wall_time, + "step": step, + "precision": data_array[ + metadata.PRECISION_INDEX, :end_index + ].tolist(), + "recall": data_array[metadata.RECALL_INDEX, :end_index].tolist(), + "true_positives": true_positives[:end_index], + "false_positives": false_positives[:end_index], + "true_negatives": true_negatives[:end_index], + "false_negatives": false_negatives[:end_index], + "thresholds": thresholds[:end_index].tolist(), + } diff --git a/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/summary.py b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/summary.py new file mode 100644 index 0000000000000000000000000000000000000000..9eed11cee9362a642bc07b773a4b5f1adc2afb8a --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/plugins/pr_curve/summary.py @@ -0,0 +1,576 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Precision--recall curves and TensorFlow operations to create them. + +NOTE: This module is in beta, and its API is subject to change, but the +data that it stores to disk will be supported forever. +""" + + +import numpy as np + +from tensorboard.plugins.pr_curve import metadata + + +# A value that we use as the minimum value during division of counts to prevent +# division by 0. 1.0 does not work: Certain weights could cause counts below 1. +_MINIMUM_COUNT = 1e-7 + +# The default number of thresholds. +_DEFAULT_NUM_THRESHOLDS = 201 + + +def op( + name, + labels, + predictions, + num_thresholds=None, + weights=None, + display_name=None, + description=None, + collections=None, +): + """Create a PR curve summary op for a single binary classifier. + + Computes true/false positive/negative values for the given `predictions` + against the ground truth `labels`, against a list of evenly distributed + threshold values in `[0, 1]` of length `num_thresholds`. + + Each number in `predictions`, a float in `[0, 1]`, is compared with its + corresponding boolean label in `labels`, and counts as a single tp/fp/tn/fn + value at each threshold. This is then multiplied with `weights` which can be + used to reweight certain values, or more commonly used for masking values. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + labels: The ground truth values. A Tensor of `bool` values with arbitrary + shape. + predictions: A float32 `Tensor` whose values are in the range `[0, 1]`. + Dimensions must match those of `labels`. + num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to + compute PR metrics for. Should be `>= 2`. This value should be a + constant integer value, not a Tensor that stores an integer. + weights: Optional float32 `Tensor`. Individual counts are multiplied by this + value. This tensor must be either the same shape as or broadcastable to + the `labels` tensor. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A summary operation for use in a TensorFlow graph. The float32 tensor + produced by the summary operation is of dimension (6, num_thresholds). The + first dimension (of length 6) is of the order: true positives, + false positives, true negatives, false negatives, precision, recall. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if num_thresholds is None: + num_thresholds = _DEFAULT_NUM_THRESHOLDS + + if weights is None: + weights = 1.0 + + dtype = predictions.dtype + + with tf.name_scope(name, values=[labels, predictions, weights]): + tf.assert_type(labels, tf.bool) + # We cast to float to ensure we have 0.0 or 1.0. + f_labels = tf.cast(labels, dtype) + # Ensure predictions are all in range [0.0, 1.0]. + predictions = tf.minimum(1.0, tf.maximum(0.0, predictions)) + # Get weighted true/false labels. + true_labels = f_labels * weights + false_labels = (1.0 - f_labels) * weights + + # Before we begin, flatten predictions. + predictions = tf.reshape(predictions, [-1]) + + # Shape the labels so they are broadcast-able for later multiplication. + true_labels = tf.reshape(true_labels, [-1, 1]) + false_labels = tf.reshape(false_labels, [-1, 1]) + + # To compute TP/FP/TN/FN, we are measuring a binary classifier + # C(t) = (predictions >= t) + # at each threshold 't'. So we have + # TP(t) = sum( C(t) * true_labels ) + # FP(t) = sum( C(t) * false_labels ) + # + # But, computing C(t) requires computation for each t. To make it fast, + # observe that C(t) is a cumulative integral, and so if we have + # thresholds = [t_0, ..., t_{n-1}]; t_0 < ... < t_{n-1} + # where n = num_thresholds, and if we can compute the bucket function + # B(i) = Sum( (predictions == t), t_i <= t < t{i+1} ) + # then we get + # C(t_i) = sum( B(j), j >= i ) + # which is the reversed cumulative sum in tf.cumsum(). + # + # We can compute B(i) efficiently by taking advantage of the fact that + # our thresholds are evenly distributed, in that + # width = 1.0 / (num_thresholds - 1) + # thresholds = [0.0, 1*width, 2*width, 3*width, ..., 1.0] + # Given a prediction value p, we can map it to its bucket by + # bucket_index(p) = floor( p * (num_thresholds - 1) ) + # so we can use tf.scatter_add() to update the buckets in one pass. + + # Compute the bucket indices for each prediction value. + bucket_indices = tf.cast( + tf.floor(predictions * (num_thresholds - 1)), tf.int32 + ) + + # Bucket predictions. + tp_buckets = tf.reduce_sum( + input_tensor=tf.one_hot(bucket_indices, depth=num_thresholds) + * true_labels, + axis=0, + ) + fp_buckets = tf.reduce_sum( + input_tensor=tf.one_hot(bucket_indices, depth=num_thresholds) + * false_labels, + axis=0, + ) + + # Set up the cumulative sums to compute the actual metrics. + tp = tf.cumsum(tp_buckets, reverse=True, name="tp") + fp = tf.cumsum(fp_buckets, reverse=True, name="fp") + # fn = sum(true_labels) - tp + # = sum(tp_buckets) - tp + # = tp[0] - tp + # Similarly, + # tn = fp[0] - fp + tn = fp[0] - fp + fn = tp[0] - tp + + precision = tp / tf.maximum(_MINIMUM_COUNT, tp + fp) + recall = tp / tf.maximum(_MINIMUM_COUNT, tp + fn) + + return _create_tensor_summary( + name, + tp, + fp, + tn, + fn, + precision, + recall, + num_thresholds, + display_name, + description, + collections, + ) + + +def pb( + name, + labels, + predictions, + num_thresholds=None, + weights=None, + display_name=None, + description=None, +): + """Create a PR curves summary protobuf. + + Arguments: + name: A name for the generated node. Will also serve as a series name in + TensorBoard. + labels: The ground truth values. A bool numpy array. + predictions: A float32 numpy array whose values are in the range `[0, 1]`. + Dimensions must match those of `labels`. + num_thresholds: Optional number of thresholds, evenly distributed in + `[0, 1]`, to compute PR metrics for. When provided, should be an int of + value at least 2. Defaults to 201. + weights: Optional float or float32 numpy array. Individual counts are + multiplied by this value. This tensor must be either the same shape as + or broadcastable to the `labels` numpy array. + display_name: Optional name for this summary in TensorBoard, as a `str`. + Defaults to `name`. + description: Optional long-form description for this summary, as a `str`. + Markdown is supported. Defaults to empty. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf # noqa: F401 + + if num_thresholds is None: + num_thresholds = _DEFAULT_NUM_THRESHOLDS + + if weights is None: + weights = 1.0 + + # Compute bins of true positives and false positives. + bucket_indices = np.int32(np.floor(predictions * (num_thresholds - 1))) + float_labels = labels.astype(float) + histogram_range = (0, num_thresholds - 1) + tp_buckets, _ = np.histogram( + bucket_indices, + bins=num_thresholds, + range=histogram_range, + weights=float_labels * weights, + ) + fp_buckets, _ = np.histogram( + bucket_indices, + bins=num_thresholds, + range=histogram_range, + weights=(1.0 - float_labels) * weights, + ) + + # Obtain the reverse cumulative sum. + tp = np.cumsum(tp_buckets[::-1])[::-1] + fp = np.cumsum(fp_buckets[::-1])[::-1] + tn = fp[0] - fp + fn = tp[0] - tp + precision = tp / np.maximum(_MINIMUM_COUNT, tp + fp) + recall = tp / np.maximum(_MINIMUM_COUNT, tp + fn) + + return raw_data_pb( + name, + true_positive_counts=tp, + false_positive_counts=fp, + true_negative_counts=tn, + false_negative_counts=fn, + precision=precision, + recall=recall, + num_thresholds=num_thresholds, + display_name=display_name, + description=description, + ) + + +def streaming_op( + name, + labels, + predictions, + num_thresholds=None, + weights=None, + metrics_collections=None, + updates_collections=None, + display_name=None, + description=None, +): + """Computes a precision-recall curve summary across batches of data. + + This function is similar to op() above, but can be used to compute the PR + curve across multiple batches of labels and predictions, in the same style + as the metrics found in tf.metrics. + + This function creates multiple local variables for storing true positives, + true negative, etc. accumulated over each batch of data, and uses these local + variables for computing the final PR curve summary. These variables can be + updated with the returned update_op. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + labels: The ground truth values, a `Tensor` whose dimensions must match + `predictions`. Will be cast to `bool`. + predictions: A floating point `Tensor` of arbitrary shape and whose values + are in the range `[0, 1]`. + num_thresholds: The number of evenly spaced thresholds to generate for + computing the PR curve. Defaults to 201. + weights: Optional `Tensor` whose rank is either 0, or the same rank as + `labels`, and must be broadcastable to `labels` (i.e., all dimensions must + be either `1`, or the same as the corresponding `labels` dimension). + metrics_collections: An optional list of collections that `auc` should be + added to. + updates_collections: An optional list of collections that `update_op` should + be added to. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + + Returns: + pr_curve: A string `Tensor` containing a single value: the + serialized PR curve Tensor summary. The summary contains a + float32 `Tensor` of dimension (6, num_thresholds). The first + dimension (of length 6) is of the order: true positives, false + positives, true negatives, false negatives, precision, recall. + update_op: An operation that updates the summary with the latest data. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if num_thresholds is None: + num_thresholds = _DEFAULT_NUM_THRESHOLDS + + thresholds = [i / float(num_thresholds - 1) for i in range(num_thresholds)] + + with tf.name_scope(name, values=[labels, predictions, weights]): + tp, update_tp = tf.metrics.true_positives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + fp, update_fp = tf.metrics.false_positives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + tn, update_tn = tf.metrics.true_negatives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + fn, update_fn = tf.metrics.false_negatives_at_thresholds( + labels=labels, + predictions=predictions, + thresholds=thresholds, + weights=weights, + ) + + def compute_summary(tp, fp, tn, fn, collections): + precision = tp / tf.maximum(_MINIMUM_COUNT, tp + fp) + recall = tp / tf.maximum(_MINIMUM_COUNT, tp + fn) + + return _create_tensor_summary( + name, + tp, + fp, + tn, + fn, + precision, + recall, + num_thresholds, + display_name, + description, + collections, + ) + + pr_curve = compute_summary(tp, fp, tn, fn, metrics_collections) + update_op = tf.group(update_tp, update_fp, update_tn, update_fn) + if updates_collections: + for collection in updates_collections: + tf.add_to_collection(collection, update_op) + + return pr_curve, update_op + + +def raw_data_op( + name, + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + num_thresholds=None, + display_name=None, + description=None, + collections=None, +): + """Create an op that collects data for visualizing PR curves. + + Unlike the op above, this one avoids computing precision, recall, and the + intermediate counts. Instead, it accepts those tensors as arguments and + relies on the caller to ensure that the calculations are correct (and the + counts yield the provided precision and recall values). + + This op is useful when a caller seeks to compute precision and recall + differently but still use the PR curves plugin. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + true_positive_counts: A rank-1 tensor of true positive counts. Must contain + `num_thresholds` elements and be castable to float32. Values correspond + to thresholds that increase from left to right (from 0 to 1). + false_positive_counts: A rank-1 tensor of false positive counts. Must + contain `num_thresholds` elements and be castable to float32. Values + correspond to thresholds that increase from left to right (from 0 to 1). + true_negative_counts: A rank-1 tensor of true negative counts. Must contain + `num_thresholds` elements and be castable to float32. Values + correspond to thresholds that increase from left to right (from 0 to 1). + false_negative_counts: A rank-1 tensor of false negative counts. Must + contain `num_thresholds` elements and be castable to float32. Values + correspond to thresholds that increase from left to right (from 0 to 1). + precision: A rank-1 tensor of precision values. Must contain + `num_thresholds` elements and be castable to float32. Values correspond + to thresholds that increase from left to right (from 0 to 1). + recall: A rank-1 tensor of recall values. Must contain `num_thresholds` + elements and be castable to float32. Values correspond to thresholds + that increase from left to right (from 0 to 1). + num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to + compute PR metrics for. Should be `>= 2`. This value should be a + constant integer value, not a Tensor that stores an integer. + display_name: Optional name for this summary in TensorBoard, as a + constant `str`. Defaults to `name`. + description: Optional long-form description for this summary, as a + constant `str`. Markdown is supported. Defaults to empty. + collections: Optional list of graph collections keys. The new + summary op is added to these collections. Defaults to + `[Graph Keys.SUMMARIES]`. + + Returns: + A summary operation for use in a TensorFlow graph. See docs for the `op` + method for details on the float32 tensor produced by this summary. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + with tf.name_scope( + name, + values=[ + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + ], + ): + return _create_tensor_summary( + name, + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + num_thresholds, + display_name, + description, + collections, + ) + + +def raw_data_pb( + name, + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + num_thresholds=None, + display_name=None, + description=None, +): + """Create a PR curves summary protobuf from raw data values. + + Args: + name: A tag attached to the summary. Used by TensorBoard for organization. + true_positive_counts: A rank-1 numpy array of true positive counts. Must + contain `num_thresholds` elements and be castable to float32. + false_positive_counts: A rank-1 numpy array of false positive counts. Must + contain `num_thresholds` elements and be castable to float32. + true_negative_counts: A rank-1 numpy array of true negative counts. Must + contain `num_thresholds` elements and be castable to float32. + false_negative_counts: A rank-1 numpy array of false negative counts. Must + contain `num_thresholds` elements and be castable to float32. + precision: A rank-1 numpy array of precision values. Must contain + `num_thresholds` elements and be castable to float32. + recall: A rank-1 numpy array of recall values. Must contain `num_thresholds` + elements and be castable to float32. + num_thresholds: Number of thresholds, evenly distributed in `[0, 1]`, to + compute PR metrics for. Should be an int `>= 2`. + display_name: Optional name for this summary in TensorBoard, as a `str`. + Defaults to `name`. + description: Optional long-form description for this summary, as a `str`. + Markdown is supported. Defaults to empty. + + Returns: + A summary operation for use in a TensorFlow graph. See docs for the `op` + method for details on the float32 tensor produced by this summary. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + if display_name is None: + display_name = name + summary_metadata = metadata.create_summary_metadata( + display_name=display_name if display_name is not None else name, + description=description or "", + num_thresholds=num_thresholds, + ) + tf_summary_metadata = tf.SummaryMetadata.FromString( + summary_metadata.SerializeToString() + ) + summary = tf.Summary() + data = np.stack( + ( + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + ) + ) + tensor = tf.make_tensor_proto(np.float32(data), dtype=tf.float32) + summary.value.add( + tag="%s/pr_curves" % name, metadata=tf_summary_metadata, tensor=tensor + ) + return summary + + +def _create_tensor_summary( + name, + true_positive_counts, + false_positive_counts, + true_negative_counts, + false_negative_counts, + precision, + recall, + num_thresholds=None, + display_name=None, + description=None, + collections=None, +): + """A private helper method for generating a tensor summary. + + We use a helper method instead of having `op` directly call `raw_data_op` + to prevent the scope of `raw_data_op` from being embedded within `op`. + + Arguments are the same as for raw_data_op. + + Returns: + A tensor summary that collects data for PR curves. + """ + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + # Store the number of thresholds within the summary metadata because + # that value is constant for all pr curve summaries with the same tag. + summary_metadata = metadata.create_summary_metadata( + display_name=display_name if display_name is not None else name, + description=description or "", + num_thresholds=num_thresholds, + ) + + # Store values within a tensor. We store them in the order: + # true positives, false positives, true negatives, false + # negatives, precision, and recall. + combined_data = tf.stack( + [ + tf.cast(true_positive_counts, tf.float32), + tf.cast(false_positive_counts, tf.float32), + tf.cast(true_negative_counts, tf.float32), + tf.cast(false_negative_counts, tf.float32), + tf.cast(precision, tf.float32), + tf.cast(recall, tf.float32), + ] + ) + + return tf.summary.tensor_summary( + name="pr_curves", + tensor=combined_data, + collections=collections, + summary_metadata=summary_metadata, + ) diff --git a/lib/python3.12/site-packages/tensorboard/program.py b/lib/python3.12/site-packages/tensorboard/program.py new file mode 100644 index 0000000000000000000000000000000000000000..e945c3095499fe3825ebdf912eca077d84239dc0 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/program.py @@ -0,0 +1,912 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for TensorBoard command line program. + +This is a lightweight module for bringing up a TensorBoard HTTP server +or emulating the `tensorboard` shell command. + +Those wishing to create custom builds of TensorBoard can use this module +by swapping out `tensorboard.main` with the custom definition that +modifies the set of plugins and static assets. + +This module does not depend on first-party plugins or the default web +server assets. Those are defined in `tensorboard.default`. +""" + + +from abc import ABCMeta +from abc import abstractmethod +import argparse +import atexit +from collections import defaultdict +import errno +import logging +import mimetypes +import os +import shlex +import signal +import socket +import sys +import threading +import time +import urllib.parse + +from absl import flags as absl_flags +from absl.flags import argparse_flags +from werkzeug import serving + +from tensorboard import manager +from tensorboard import version +from tensorboard.backend import application +from tensorboard.backend.event_processing import data_ingester as local_ingester +from tensorboard.backend.event_processing import event_file_inspector as efi +from tensorboard.data import server_ingester +from tensorboard.plugins.core import core_plugin +from tensorboard.util import tb_logging + + +logger = tb_logging.get_logger() + +# Default subcommand name. This is a user-facing CLI and should not change. +_SERVE_SUBCOMMAND_NAME = "serve" +# Internal flag name used to store which subcommand was invoked. +_SUBCOMMAND_FLAG = "__tensorboard_subcommand" + +# Message printed when we actually use the data server, so that users are not +# caught terribly by surprise. +_DATA_SERVER_ADVISORY_MESSAGE = """ +NOTE: Using experimental fast data loading logic. To disable, pass + "--load_fast=false" and report issues on GitHub. More details: + https://github.com/tensorflow/tensorboard/issues/4784 + +""" + +# Message printed with `--load_fast=true` if the data server could not start up. +# To be formatted with one `DataServerStartupError` interpoland. +_DATA_SERVER_STARTUP_ERROR_MESSAGE_TEMPLATE = """\ +Could not start data server: %s. + Try with --load_fast=false and report issues on GitHub. Details: + https://github.com/tensorflow/tensorboard/issues/4784 +""" + + +class TensorBoard: + """Class for running TensorBoard. + + Fields: + plugin_loaders: Set from plugins passed to constructor. + assets_zip_provider: Set by constructor. + server_class: Set by constructor. + flags: An argparse.Namespace set by the configure() method. + cache_key: As `manager.cache_key`; set by the configure() method. + """ + + def __init__( + self, + plugins=None, + assets_zip_provider=None, + server_class=None, + subcommands=None, + ): + """Creates new instance. + + Args: + plugins: A list of TensorBoard plugins to load, as TBPlugin classes or + TBLoader instances or classes. If not specified, defaults to first-party + plugins. + assets_zip_provider: A function that provides a zip file containing + assets to the application. If `None`, the default TensorBoard web + assets will be used. (If building from source, your binary must + explicitly depend on `//tensorboard:assets_lib` if you pass `None`.) + server_class: An optional factory for a `TensorBoardServer` to use + for serving the TensorBoard WSGI app. If provided, its callable + signature should match that of `TensorBoardServer.__init__`. + subcommands: An optional list of TensorBoardSubcommand objects, which + extend the functionality of the CLI. + + :type plugins: + list[ + base_plugin.TBLoader | Type[base_plugin.TBLoader] | + Type[base_plugin.TBPlugin] + ] + """ + if plugins is None: + from tensorboard import default + + plugins = default.get_plugins() + if assets_zip_provider is None: + try: + from tensorboard import assets + except ImportError as e: + # `tensorboard.assets` is not a strict Bazel dep; clients are + # required to either depend on `//tensorboard:assets_lib` or + # pass a valid assets provider. + raise ImportError( + "No `assets_zip_provider` given, but `tensorboard.assets` " + "could not be imported to resolve defaults" + ) from e + assets_zip_provider = assets.get_default_assets_zip_provider() + if server_class is None: + server_class = create_port_scanning_werkzeug_server + if subcommands is None: + subcommands = [] + self.plugin_loaders = [ + application.make_plugin_loader(p) for p in plugins + ] + self.assets_zip_provider = assets_zip_provider + self.server_class = server_class + self.subcommands = {} + for subcommand in subcommands: + name = subcommand.name() + if name in self.subcommands or name == _SERVE_SUBCOMMAND_NAME: + raise ValueError("Duplicate subcommand name: %r" % name) + self.subcommands[name] = subcommand + self.flags = None + + def configure(self, argv=("",), **kwargs): + """Configures TensorBoard behavior via flags. + + This method will populate the "flags" property with an argparse.Namespace + representing flag values parsed from the provided argv list, overridden by + explicit flags from remaining keyword arguments. + + Args: + argv: Can be set to CLI args equivalent to sys.argv; the first arg is + taken to be the name of the path being executed. + kwargs: Additional arguments will override what was parsed from + argv. They must be passed as Python data structures, e.g. + `foo=1` rather than `foo="1"`. + + Returns: + Either argv[:1] if argv was non-empty, or [''] otherwise, as a mechanism + for absl.app.run() compatibility. + + Raises: + ValueError: If flag values are invalid. + """ + + base_parser = argparse_flags.ArgumentParser( + prog="tensorboard", + description=( + "TensorBoard is a suite of web applications for " + "inspecting and understanding your TensorFlow runs " + "and graphs. https://github.com/tensorflow/tensorboard " + ), + ) + subparsers = base_parser.add_subparsers( + help="TensorBoard subcommand (defaults to %r)" + % _SERVE_SUBCOMMAND_NAME + ) + + serve_subparser = subparsers.add_parser( + _SERVE_SUBCOMMAND_NAME, + help="start local TensorBoard server (default subcommand)", + ) + serve_subparser.set_defaults( + **{_SUBCOMMAND_FLAG: _SERVE_SUBCOMMAND_NAME} + ) + + if len(argv) < 2 or argv[1].startswith("-"): + # This invocation, if valid, must not use any subcommands: we + # don't permit flags before the subcommand name. + serve_parser = base_parser + else: + # This invocation, if valid, must use a subcommand: we don't take + # any positional arguments to `serve`. + serve_parser = serve_subparser + + for name, subcommand in self.subcommands.items(): + subparser = subparsers.add_parser( + name, + help=subcommand.help(), + description=subcommand.description(), + ) + subparser.set_defaults(**{_SUBCOMMAND_FLAG: name}) + subcommand.define_flags(subparser) + + for loader in self.plugin_loaders: + loader.define_flags(serve_parser) + + arg0 = argv[0] if argv else "" + + flags = base_parser.parse_args(argv[1:]) # Strip binary name from argv. + if getattr(flags, _SUBCOMMAND_FLAG, None) is None: + # Manually assign default value rather than using `set_defaults` + # on the base parser to work around Python bug #9351 on old + # versions of `argparse`: + setattr(flags, _SUBCOMMAND_FLAG, _SERVE_SUBCOMMAND_NAME) + + self.cache_key = manager.cache_key( + working_directory=os.getcwd(), + arguments=argv[1:], + configure_kwargs=kwargs, + ) + if arg0: + # Only expose main module Abseil flags as TensorBoard native flags. + # This is the same logic Abseil's ArgumentParser uses for determining + # which Abseil flags to include in the short helpstring. + for flag in set(absl_flags.FLAGS.get_key_flags_for_module(arg0)): + if hasattr(flags, flag.name): + raise ValueError("Conflicting Abseil flag: %s" % flag.name) + setattr(flags, flag.name, flag.value) + for k, v in kwargs.items(): + if not hasattr(flags, k): + raise ValueError("Unknown TensorBoard flag: %s" % k) + setattr(flags, k, v) + if getattr(flags, _SUBCOMMAND_FLAG) == _SERVE_SUBCOMMAND_NAME: + for loader in self.plugin_loaders: + loader.fix_flags(flags) + self.flags = flags + return [arg0] + + def main(self, ignored_argv=("",)): + """Blocking main function for TensorBoard. + + This method is called by `tensorboard.main.run_main`, which is the + standard entrypoint for the tensorboard command line program. The + configure() method must be called first. + + Args: + ignored_argv: Do not pass. Required for Abseil compatibility. + + Returns: + Process exit code, i.e. 0 if successful or non-zero on failure. In + practice, an exception will most likely be raised instead of + returning non-zero. + + :rtype: int + """ + self._install_signal_handler(signal.SIGTERM, "SIGTERM") + self._fix_mime_types() + subcommand_name = getattr(self.flags, _SUBCOMMAND_FLAG) + if subcommand_name == _SERVE_SUBCOMMAND_NAME: + runner = self._run_serve_subcommand + else: + runner = self.subcommands[subcommand_name].run + return runner(self.flags) or 0 + + def _run_serve_subcommand(self, flags): + # TODO(#2801): Make `--version` a flag on only the base parser, not `serve`. + if flags.version_tb: + print(version.VERSION) + return 0 + if flags.inspect: + # TODO(@wchargin): Convert `inspect` to a normal subcommand? + logger.info( + "Not bringing up TensorBoard, but inspecting event files." + ) + event_file = os.path.expanduser(flags.event_file) + efi.inspect(flags.logdir, event_file, flags.tag) + return 0 + try: + server = self._make_server() + server.print_serving_message() + self._register_info(server) + server.serve_forever() + return 0 + except TensorBoardServerException as e: + logger.error(e.msg) + sys.stderr.write("ERROR: %s\n" % e.msg) + sys.stderr.flush() + return -1 + + def launch(self): + """Python API for launching TensorBoard. + + This method is the same as main() except it launches TensorBoard in + a separate permanent thread. The configure() method must be called + first. + + Returns: + The URL of the TensorBoard web server. + + :rtype: str + """ + # Make it easy to run TensorBoard inside other programs, e.g. Colab. + server = self._make_server() + thread = threading.Thread( + target=server.serve_forever, name="TensorBoard" + ) + thread.daemon = True + thread.start() + return server.get_url() + + def _register_info(self, server): + """Write a TensorBoardInfo file and arrange for its cleanup. + + Args: + server: The result of `self._make_server()`. + """ + server_url = urllib.parse.urlparse(server.get_url()) + info = manager.TensorBoardInfo( + version=version.VERSION, + start_time=int(time.time()), + port=server_url.port, + pid=os.getpid(), + path_prefix=self.flags.path_prefix, + logdir=self.flags.logdir or self.flags.logdir_spec, + db=self.flags.db, + cache_key=self.cache_key, + ) + atexit.register(manager.remove_info_file) + manager.write_info_file(info) + + def _install_signal_handler(self, signal_number, signal_name): + """Set a signal handler to gracefully exit on the given signal. + + When this process receives the given signal, it will run `atexit` + handlers and then exit with `0`. + + Args: + signal_number: The numeric code for the signal to handle, like + `signal.SIGTERM`. + signal_name: The human-readable signal name. + """ + # Note to maintainers: Google-internal code overrides this + # method (cf. cl/334534610). Double-check changes before + # modifying API. + + old_signal_handler = None # set below + + def handler(handled_signal_number, frame): + # In case we catch this signal again while running atexit + # handlers, take the hint and actually die. + signal.signal(signal_number, signal.SIG_DFL) + sys.stderr.write( + "TensorBoard caught %s; exiting...\n" % signal_name + ) + # The main thread is the only non-daemon thread, so it suffices to + # exit hence. + if old_signal_handler not in (signal.SIG_IGN, signal.SIG_DFL): + old_signal_handler(handled_signal_number, frame) + sys.exit(0) + + old_signal_handler = signal.signal(signal_number, handler) + + def _fix_mime_types(self): + """Fix incorrect entries in the `mimetypes` registry. + + On Windows, the Python standard library's `mimetypes` reads in + mappings from file extension to MIME type from the Windows + registry. Other applications can and do write incorrect values + to this registry, which causes `mimetypes.guess_type` to return + incorrect values, which causes TensorBoard to fail to render on + the frontend. + + This method hard-codes the correct mappings for certain MIME + types that are known to be either used by TensorBoard or + problematic in general. + """ + # Known to be problematic when Visual Studio is installed: + # + mimetypes.add_type("text/javascript", ".js") + # Not known to be problematic, but used by TensorBoard: + mimetypes.add_type("font/woff2", ".woff2") + mimetypes.add_type("text/html", ".html") + + def _start_subprocess_data_ingester(self): + """Creates, starts, and returns a `SubprocessServerDataIngester`.""" + flags = self.flags + server_binary = server_ingester.get_server_binary() + ingester = server_ingester.SubprocessServerDataIngester( + server_binary=server_binary, + logdir=flags.logdir, + reload_interval=flags.reload_interval, + channel_creds_type=flags.grpc_creds_type, + samples_per_plugin=flags.samples_per_plugin, + extra_flags=shlex.split(flags.extra_data_server_flags), + ) + ingester.start() + return ingester + + def _make_data_ingester(self): + """Determines the right data ingester, starts it, and returns it.""" + flags = self.flags + if flags.grpc_data_provider: + ingester = server_ingester.ExistingServerDataIngester( + flags.grpc_data_provider, + channel_creds_type=flags.grpc_creds_type, + ) + ingester.start() + return ingester + + if flags.load_fast == "true": + try: + return self._start_subprocess_data_ingester() + except server_ingester.NoDataServerError as e: + msg = "Option --load_fast=true not available: %s\n" % e + sys.stderr.write(msg) + sys.exit(1) + except server_ingester.DataServerStartupError as e: + msg = _DATA_SERVER_STARTUP_ERROR_MESSAGE_TEMPLATE % e + sys.stderr.write(msg) + sys.exit(1) + + if flags.load_fast == "auto" and _should_use_data_server(flags): + try: + ingester = self._start_subprocess_data_ingester() + sys.stderr.write(_DATA_SERVER_ADVISORY_MESSAGE) + sys.stderr.flush() + return ingester + except server_ingester.NoDataServerError as e: + logger.info("No data server: %s", e) + except server_ingester.DataServerStartupError as e: + logger.info( + "Data server error: %s; falling back to multiplexer", e + ) + + ingester = local_ingester.LocalDataIngester(flags) + ingester.start() + return ingester + + def _make_data_provider(self): + """Returns `(data_provider, deprecated_multiplexer)`.""" + ingester = self._make_data_ingester() + # Stash ingester so that it can avoid GCing Windows file handles. + # (See comment in `SubprocessServerDataIngester.start` for details.) + self._ingester = ingester + + deprecated_multiplexer = None + if isinstance(ingester, local_ingester.LocalDataIngester): + deprecated_multiplexer = ingester.deprecated_multiplexer + return (ingester.data_provider, deprecated_multiplexer) + + def _make_server(self): + """Constructs the TensorBoard WSGI app and instantiates the server.""" + (data_provider, deprecated_multiplexer) = self._make_data_provider() + app = application.TensorBoardWSGIApp( + self.flags, + self.plugin_loaders, + data_provider, + self.assets_zip_provider, + deprecated_multiplexer, + ) + return self.server_class(app, self.flags) + + +def _should_use_data_server(flags): + if flags.logdir_spec and not flags.logdir: + logger.info( + "Note: --logdir_spec is not supported with --load_fast behavior; " + "falling back to slower Python-only load path. To use the data " + "server, replace --logdir_spec with --logdir." + ) + return False + if not flags.logdir: + # Using some other legacy mode; not supported. + return False + if "://" in flags.logdir and not flags.logdir.startswith("gs://"): + logger.info( + "Note: --load_fast behavior only supports local and GCS (gs://) " + "paths; falling back to slower Python-only load path." + ) + return False + if flags.detect_file_replacement is True: + logger.info( + "Note: --detect_file_replacement=true is not supported with " + "--load_fast behavior; falling back to slower Python-only load " + "path." + ) + return False + return True + + +class TensorBoardSubcommand(metaclass=ABCMeta): + """Experimental private API for defining subcommands for tensorboard. + + The intended use is something like: + + `tensorboard ` + + Since our hosted service at http://tensorboard.dev has been shut down, this + functionality is no longer used, but the support for subcommands remains, + in case it is ever useful again. + """ + + @abstractmethod + def name(self): + """Name of this subcommand, as specified on the command line. + + This must be unique across all subcommands. + + Returns: + A string. + """ + pass + + @abstractmethod + def define_flags(self, parser): + """Configure an argument parser for this subcommand. + + Flags whose names start with two underscores (e.g., `__foo`) are + reserved for use by the runtime and must not be defined by + subcommands. + + Args: + parser: An `argparse.ArgumentParser` scoped to this subcommand, + which this function should mutate. + """ + pass + + @abstractmethod + def run(self, flags): + """Execute this subcommand with user-provided flags. + + Args: + flags: An `argparse.Namespace` object with all defined flags. + + Returns: + An `int` exit code, or `None` as an alias for `0`. + """ + pass + + def help(self): + """Short, one-line help text to display on `tensorboard --help`.""" + return None + + def description(self): + """Description to display on `tensorboard SUBCOMMAND --help`.""" + return None + + +class TensorBoardServer(metaclass=ABCMeta): + """Class for customizing TensorBoard WSGI app serving.""" + + @abstractmethod + def __init__(self, wsgi_app, flags): + """Create a flag-configured HTTP server for TensorBoard's WSGI app. + + Args: + wsgi_app: The TensorBoard WSGI application to create a server for. + flags: argparse.Namespace instance of TensorBoard flags. + """ + raise NotImplementedError() + + @abstractmethod + def serve_forever(self): + """Blocking call to start serving the TensorBoard server.""" + raise NotImplementedError() + + @abstractmethod + def get_url(self): + """Returns a URL at which this server should be reachable.""" + raise NotImplementedError() + + def print_serving_message(self): + """Prints a user-friendly message prior to server start. + + This will be called just before `serve_forever`. + """ + sys.stderr.write( + "TensorBoard %s at %s (Press CTRL+C to quit)\n" + % (version.VERSION, self.get_url()) + ) + sys.stderr.flush() + + +class TensorBoardServerException(Exception): + """Exception raised by TensorBoardServer for user-friendly errors. + + Subclasses of TensorBoardServer can raise this exception in order to + generate a clean error message for the user rather than a + stacktrace. + """ + + def __init__(self, msg): + self.msg = msg + + +class TensorBoardPortInUseError(TensorBoardServerException): + """Error raised when attempting to bind to a port that is in use. + + This should be raised when it is expected that binding to another + similar port would succeed. It is used as a signal to indicate that + automatic port searching should continue rather than abort. + """ + + pass + + +def with_port_scanning(cls): + """Create a server factory that performs port scanning. + + This function returns a callable whose signature matches the + specification of `TensorBoardServer.__init__`, using `cls` as an + underlying implementation. It passes through `flags` unchanged except + in the case that `flags.port is None`, in which case it repeatedly + instantiates the underlying server with new port suggestions. + + Args: + cls: A valid implementation of `TensorBoardServer`. This class's + initializer should raise a `TensorBoardPortInUseError` upon + failing to bind to a port when it is expected that binding to + another nearby port might succeed. + + The initializer for `cls` will only ever be invoked with `flags` + such that `flags.port is not None`. + + Returns: + A function that implements the `__init__` contract of + `TensorBoardServer`. + """ + + def init(wsgi_app, flags): + # base_port: what's the first port to which we should try to bind? + # should_scan: if that fails, shall we try additional ports? + # max_attempts: how many ports shall we try? + should_scan = flags.port is None + base_port = ( + core_plugin.DEFAULT_PORT if flags.port is None else flags.port + ) + + if base_port > 0xFFFF: + raise TensorBoardServerException( + "TensorBoard cannot bind to port %d > %d" % (base_port, 0xFFFF) + ) + max_attempts = 100 if should_scan else 1 + base_port = min(base_port + max_attempts, 0x10000) - max_attempts + + for port in range(base_port, base_port + max_attempts): + subflags = argparse.Namespace(**vars(flags)) + subflags.port = port + try: + return cls(wsgi_app=wsgi_app, flags=subflags) + except TensorBoardPortInUseError: + if not should_scan: + raise + # All attempts failed to bind. + raise TensorBoardServerException( + "TensorBoard could not bind to any port around %s " + "(tried %d times)" % (base_port, max_attempts) + ) + + return init + + +class _WSGIRequestHandler(serving.WSGIRequestHandler): + """Custom subclass of Werkzeug request handler to use HTTP/1.1.""" + + # The default on the http.server is HTTP/1.0 for legacy reasons: + # https://docs.python.org/3/library/http.server.html#http.server.BaseHTTPRequestHandler.protocol_version + # Override here to use HTTP/1.1 to avoid needing a new TCP socket and Python + # thread for each HTTP request. The tradeoff is we must always specify the + # Content-Length header, or do chunked encoding for streaming. + protocol_version = "HTTP/1.1" + + +class WerkzeugServer(serving.ThreadedWSGIServer, TensorBoardServer): + """Implementation of TensorBoardServer using the Werkzeug dev server.""" + + # ThreadedWSGIServer handles this in werkzeug 0.12+ but we allow 0.11.x. + daemon_threads = True + + def __init__(self, wsgi_app, flags): + self._flags = flags + host = flags.host + port = flags.port + + self._auto_wildcard = flags.bind_all + if self._auto_wildcard: + # Serve on all interfaces, and attempt to serve both IPv4 and IPv6 + # traffic through one socket. + host = self._get_wildcard_address(port) + elif host is None: + host = "localhost" + + self._host = host + self._url = None # Will be set by get_url() below + + self._fix_werkzeug_logging() + + def is_port_in_use(port): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(("localhost", port)) == 0 + + try: + if is_port_in_use(port): + raise TensorBoardPortInUseError( + "TensorBoard could not bind to port %d, it was already in use" + % port + ) + super().__init__(host, port, wsgi_app, _WSGIRequestHandler) + except socket.error as e: + if hasattr(errno, "EACCES") and e.errno == errno.EACCES: + raise TensorBoardServerException( + "TensorBoard must be run as superuser to bind to port %d" + % port + ) + elif hasattr(errno, "EADDRINUSE") and e.errno == errno.EADDRINUSE: + if port == 0: + raise TensorBoardServerException( + "TensorBoard unable to find any open port" + ) + else: + raise TensorBoardPortInUseError( + "TensorBoard could not bind to port %d, it was already in use" + % port + ) + elif ( + hasattr(errno, "EADDRNOTAVAIL") + and e.errno == errno.EADDRNOTAVAIL + ): + raise TensorBoardServerException( + "TensorBoard could not bind to unavailable address %s" + % host + ) + elif ( + hasattr(errno, "EAFNOSUPPORT") and e.errno == errno.EAFNOSUPPORT + ): + raise TensorBoardServerException( + "Tensorboard could not bind to unsupported address family %s" + % host + ) + # Raise the raw exception if it wasn't identifiable as a user error. + raise + + def _get_wildcard_address(self, port): + """Returns a wildcard address for the port in question. + + This will attempt to follow the best practice of calling + getaddrinfo() with a null host and AI_PASSIVE to request a + server-side socket wildcard address. If that succeeds, this + returns the first IPv6 address found, or if none, then returns + the first IPv4 address. If that fails, then this returns the + hardcoded address "::" if socket.has_ipv6 is True, else + "0.0.0.0". + """ + fallback_address = "::" if socket.has_ipv6 else "0.0.0.0" + if hasattr(socket, "AI_PASSIVE"): + try: + addrinfos = socket.getaddrinfo( + None, + port, + socket.AF_UNSPEC, + socket.SOCK_STREAM, + socket.IPPROTO_TCP, + socket.AI_PASSIVE, + ) + except socket.gaierror as e: + logger.warning( + "Failed to auto-detect wildcard address, assuming %s: %s", + fallback_address, + str(e), + ) + return fallback_address + addrs_by_family = defaultdict(list) + for family, _, _, _, sockaddr in addrinfos: + # Format of the "sockaddr" socket address varies by address family, + # but [0] is always the IP address portion. + addrs_by_family[family].append(sockaddr[0]) + if hasattr(socket, "AF_INET6") and addrs_by_family[socket.AF_INET6]: + return addrs_by_family[socket.AF_INET6][0] + if hasattr(socket, "AF_INET") and addrs_by_family[socket.AF_INET]: + return addrs_by_family[socket.AF_INET][0] + logger.warning( + "Failed to auto-detect wildcard address, assuming %s", + fallback_address, + ) + return fallback_address + + def server_bind(self): + """Override to set custom options on the socket.""" + if self._flags.reuse_port: + try: + socket.SO_REUSEPORT + except AttributeError: + raise TensorBoardServerException( + "TensorBoard --reuse_port option is not supported on this platform" + ) + self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) + + # Enable IPV4 mapping for IPV6 sockets when desired. + # The main use case for this is so that when no host is specified, + # TensorBoard can listen on all interfaces for both IPv4 and IPv6 + # connections, rather than having to choose v4 or v6 and hope the + # browser didn't choose the other one. + socket_is_v6 = ( + hasattr(socket, "AF_INET6") + and self.socket.family == socket.AF_INET6 + ) + has_v6only_option = hasattr(socket, "IPPROTO_IPV6") and hasattr( + socket, "IPV6_V6ONLY" + ) + if self._auto_wildcard and socket_is_v6 and has_v6only_option: + try: + self.socket.setsockopt( + socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0 + ) + except socket.error as e: + # Log a warning on failure to dual-bind, except for EAFNOSUPPORT + # since that's expected if IPv4 isn't supported at all (IPv6-only). + if ( + hasattr(errno, "EAFNOSUPPORT") + and e.errno != errno.EAFNOSUPPORT + ): + logger.warning( + "Failed to dual-bind to IPv4 wildcard: %s", str(e) + ) + super().server_bind() + + def handle_error(self, request, client_address): + """Override to get rid of noisy EPIPE errors.""" + del request # unused + # Kludge to override a SocketServer.py method so we can get rid of noisy + # EPIPE errors. They're kind of a red herring as far as errors go. For + # example, `curl -N http://localhost:6006/ | head` will cause an EPIPE. + exc_info = sys.exc_info() + e = exc_info[1] + if isinstance(e, IOError) and e.errno == errno.EPIPE: + logger.warning( + "EPIPE caused by %s in HTTP serving" % str(client_address) + ) + else: + logger.error("HTTP serving error", exc_info=exc_info) + + def get_url(self): + if not self._url: + if self._auto_wildcard: + display_host = socket.getfqdn() + # Confirm that the connection is open, otherwise change to `localhost` + try: + socket.create_connection( + (display_host, self.server_port), timeout=1 + ) + except socket.error as e: + display_host = "localhost" + + else: + host = self._host + display_host = ( + "[%s]" % host + if ":" in host and not host.startswith("[") + else host + ) + self._url = "http://%s:%d%s/" % ( + display_host, + self.server_port, + self._flags.path_prefix.rstrip("/"), + ) + return self._url + + def print_serving_message(self): + if self._flags.host is None and not self._flags.bind_all: + sys.stderr.write( + "Serving TensorBoard on localhost; to expose to the network, " + "use a proxy or pass --bind_all\n" + ) + sys.stderr.flush() + super().print_serving_message() + + def _fix_werkzeug_logging(self): + """Fix werkzeug logging setup so it inherits TensorBoard's log level. + + This addresses a change in werkzeug 0.15.0+ [1] that causes it set its own + log level to INFO regardless of the root logger configuration. We instead + want werkzeug to inherit TensorBoard's root logger log level (set via absl + to WARNING by default). + + [1]: https://github.com/pallets/werkzeug/commit/4cf77d25858ff46ac7e9d64ade054bf05b41ce12 + """ + # Log once at DEBUG to force werkzeug to initialize its singleton logger, + # which sets the logger level to INFO it if is unset, and then access that + # object via logging.getLogger('werkzeug') to durably revert the level to + # unset (and thus make messages logged to it inherit the root logger level). + self.log( + "debug", "Fixing werkzeug logger to inherit TensorBoard log level" + ) + logging.getLogger("werkzeug").setLevel(logging.NOTSET) + + +create_port_scanning_werkzeug_server = with_port_scanning(WerkzeugServer) diff --git a/lib/python3.12/site-packages/tensorboard/util/__init__.py b/lib/python3.12/site-packages/tensorboard/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35c06b059ecba46b333a9575a816aa39364f6088 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/encoder.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/encoder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f23be15486cb60091360b5fd9988197989b5ad10 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/encoder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/grpc_util.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/grpc_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae3971a75800b3009a6491e45b3830af5889902a Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/grpc_util.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/io_util.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/io_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e26833aa3843a9080870d738edb28b149405f8c Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/io_util.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/lazy_tensor_creator.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/lazy_tensor_creator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e332214501f79390a35dad72a0cb18d4c9ab3523 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/lazy_tensor_creator.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/op_evaluator.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/op_evaluator.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ad18a40da8dd55cff70676c414f07dcd0bc03f6a Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/op_evaluator.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/platform_util.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/platform_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0bbce4e6489116ea2bf3291c42a94c6cdd4b980f Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/platform_util.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/tb_logging.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/tb_logging.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..31a4c1ca83e566bb8221a14c4698eb6d492de9de Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/tb_logging.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/tensor_util.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/tensor_util.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e9391d7bfbf0eb91082b7cc03ba0b1cc0caf05a7 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/tensor_util.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/__pycache__/timing.cpython-312.pyc b/lib/python3.12/site-packages/tensorboard/util/__pycache__/timing.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbfbc469355a8477960b84487f20b9b76f0b4df4 Binary files /dev/null and b/lib/python3.12/site-packages/tensorboard/util/__pycache__/timing.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tensorboard/util/encoder.py b/lib/python3.12/site-packages/tensorboard/util/encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..55970562470de259e8fcfece317a2c6b8ff83799 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/encoder.py @@ -0,0 +1,117 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TensorBoard encoder helper module. + +Encoder depends on TensorFlow. +""" + + +import numpy as np + +from tensorboard.util import op_evaluator + + +class _TensorFlowPngEncoder(op_evaluator.PersistentOpEvaluator): + """Encode an image to PNG. + + This function is thread-safe, and has high performance when run in + parallel. See `encode_png_benchmark.py` for details. + + Arguments: + image: A numpy array of shape `[height, width, channels]`, where + `channels` is 1, 3, or 4, and of dtype uint8. + + Returns: + A bytestring with PNG-encoded data. + """ + + def __init__(self): + super().__init__() + self._image_placeholder = None + self._encode_op = None + + def initialize_graph(self): + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + self._image_placeholder = tf.placeholder( + dtype=tf.uint8, name="image_to_encode" + ) + self._encode_op = tf.image.encode_png(self._image_placeholder) + + def run(self, image): # pylint: disable=arguments-differ + if not isinstance(image, np.ndarray): + raise ValueError("'image' must be a numpy array: %r" % image) + if image.dtype != np.uint8: + raise ValueError( + "'image' dtype must be uint8, but is %r" % image.dtype + ) + return self._encode_op.eval(feed_dict={self._image_placeholder: image}) + + +encode_png = _TensorFlowPngEncoder() + + +class _TensorFlowWavEncoder(op_evaluator.PersistentOpEvaluator): + """Encode an audio clip to WAV. + + This function is thread-safe and exhibits good parallel performance. + + Arguments: + audio: A numpy array of shape `[samples, channels]`. + samples_per_second: A positive `int`, in Hz. + + Returns: + A bytestring with WAV-encoded data. + """ + + def __init__(self): + super().__init__() + self._audio_placeholder = None + self._samples_per_second_placeholder = None + self._encode_op = None + + def initialize_graph(self): + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + self._audio_placeholder = tf.placeholder( + dtype=tf.float32, name="image_to_encode" + ) + self._samples_per_second_placeholder = tf.placeholder( + dtype=tf.int32, name="samples_per_second" + ) + self._encode_op = tf.audio.encode_wav( + self._audio_placeholder, + sample_rate=self._samples_per_second_placeholder, + ) + + def run( + self, audio, samples_per_second + ): # pylint: disable=arguments-differ + if not isinstance(audio, np.ndarray): + raise ValueError("'audio' must be a numpy array: %r" % audio) + if not isinstance(samples_per_second, int): + raise ValueError( + "'samples_per_second' must be an int: %r" % samples_per_second + ) + feed_dict = { + self._audio_placeholder: audio, + self._samples_per_second_placeholder: samples_per_second, + } + return self._encode_op.eval(feed_dict=feed_dict) + + +encode_wav = _TensorFlowWavEncoder() diff --git a/lib/python3.12/site-packages/tensorboard/util/grpc_util.py b/lib/python3.12/site-packages/tensorboard/util/grpc_util.py new file mode 100644 index 0000000000000000000000000000000000000000..f05d69e18110bef5d71d7f351bcada03286a959d --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/grpc_util.py @@ -0,0 +1,312 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for working with python gRPC stubs.""" + + +import enum +import functools +import random +import threading +import time + +import grpc + +from tensorboard import version +from tensorboard.util import tb_logging + +logger = tb_logging.get_logger() + +# Default RPC timeout. +_GRPC_DEFAULT_TIMEOUT_SECS = 30 + +# Max number of times to attempt an RPC, retrying on transient failures. +_GRPC_RETRY_MAX_ATTEMPTS = 5 + +# Parameters to control the exponential backoff behavior. +_GRPC_RETRY_EXPONENTIAL_BASE = 2 +_GRPC_RETRY_JITTER_FACTOR_MIN = 1.1 +_GRPC_RETRY_JITTER_FACTOR_MAX = 1.5 + +# Status codes from gRPC for which it's reasonable to retry the RPC. +_GRPC_RETRYABLE_STATUS_CODES = frozenset( + [ + grpc.StatusCode.ABORTED, + grpc.StatusCode.DEADLINE_EXCEEDED, + grpc.StatusCode.RESOURCE_EXHAUSTED, + grpc.StatusCode.UNAVAILABLE, + ] +) + +# gRPC metadata key whose value contains the client version. +_VERSION_METADATA_KEY = "tensorboard-version" + + +class AsyncCallFuture: + """Encapsulates the future value of a retriable async gRPC request. + + Abstracts over the set of futures returned by a set of gRPC calls + comprising a single logical gRPC request with retries. Communicates + to the caller the result or exception resulting from the request. + + Args: + completion_event: The constructor should provide a `threding.Event` which + will be used to communicate when the set of gRPC requests is complete. + """ + + def __init__(self, completion_event): + self._active_grpc_future = None + self._active_grpc_future_lock = threading.Lock() + self._completion_event = completion_event + + def _set_active_future(self, grpc_future): + if grpc_future is None: + raise RuntimeError( + "_set_active_future invoked with grpc_future=None." + ) + with self._active_grpc_future_lock: + self._active_grpc_future = grpc_future + + def result(self, timeout): + """Analogous to `grpc.Future.result`. Returns the value or exception. + + This method will wait until the full set of gRPC requests is complete + and then act as `grpc.Future.result` for the single gRPC invocation + corresponding to the first successful call or final failure, as + appropriate. + + Args: + timeout: How long to wait in seconds before giving up and raising. + + Returns: + The result of the future corresponding to the single gRPC + corresponding to the successful call. + + Raises: + * `grpc.FutureTimeoutError` if timeout seconds elapse before the gRPC + calls could complete, including waits and retries. + * The exception corresponding to the last non-retryable gRPC request + in the case that a successful gRPC request was not made. + """ + if not self._completion_event.wait(timeout): + raise grpc.FutureTimeoutError( + f"AsyncCallFuture timed out after {timeout} seconds" + ) + with self._active_grpc_future_lock: + if self._active_grpc_future is None: + raise RuntimeError("AsyncFuture never had an active future set") + return self._active_grpc_future.result() + + +def async_call_with_retries(api_method, request, clock=None): + """Initiate an asynchronous call to a gRPC stub, with retry logic. + + This is similar to the `async_call` API, except that the call is handled + asynchronously, and the completion may be handled by another thread. The + caller must provide a `done_callback` argument which will handle the + result or exception rising from the gRPC completion. + + Retries are handled with jittered exponential backoff to spread out failures + due to request spikes. + + This only supports unary-unary RPCs: i.e., no streaming on either end. + + Args: + api_method: Callable for the API method to invoke. + request: Request protocol buffer to pass to the API method. + clock: an interface object supporting `time()` and `sleep()` methods + like the standard `time` module; if not passed, uses the normal module. + + Returns: + An `AsyncCallFuture` which will encapsulate the `grpc.Future` + corresponding to the gRPC call which either completes successfully or + represents the final try. + """ + if clock is None: + clock = time + logger.debug("Async RPC call %s with request: %r", api_method, request) + + completion_event = threading.Event() + async_future = AsyncCallFuture(completion_event) + + def async_call(handler): + """Invokes the gRPC future and orchestrates it via the AsyncCallFuture.""" + future = api_method.future( + request, + timeout=_GRPC_DEFAULT_TIMEOUT_SECS, + metadata=version_metadata(), + ) + # Ensure we set the active future before invoking the done callback, to + # avoid the case where the done callback completes immediately and + # triggers completion event while async_future still holds the old + # future. + async_future._set_active_future(future) + future.add_done_callback(handler) + + # retry_handler is the continuation of the `async_call`. It should: + # * If the grpc call succeeds: trigger the `completion_event`. + # * If there are no more retries: trigger the `completion_event`. + # * Otherwise, invoke a new async_call with the same + # retry_handler. + def retry_handler(future, num_attempts): + e = future.exception() + if e is None: + completion_event.set() + return + else: + logger.info("RPC call %s got error %s", api_method, e) + # If unable to retry, proceed to completion. + if e.code() not in _GRPC_RETRYABLE_STATUS_CODES: + completion_event.set() + return + if num_attempts >= _GRPC_RETRY_MAX_ATTEMPTS: + completion_event.set() + return + # If able to retry, wait then do so. + backoff_secs = _compute_backoff_seconds(num_attempts) + clock.sleep(backoff_secs) + async_call( + functools.partial(retry_handler, num_attempts=num_attempts + 1) + ) + + async_call(functools.partial(retry_handler, num_attempts=1)) + return async_future + + +def _compute_backoff_seconds(num_attempts): + """Compute appropriate wait time between RPC attempts.""" + jitter_factor = random.uniform( + _GRPC_RETRY_JITTER_FACTOR_MIN, _GRPC_RETRY_JITTER_FACTOR_MAX + ) + backoff_secs = (_GRPC_RETRY_EXPONENTIAL_BASE**num_attempts) * jitter_factor + return backoff_secs + + +def call_with_retries(api_method, request, clock=None): + """Call a gRPC stub API method, with automatic retry logic. + + This only supports unary-unary RPCs: i.e., no streaming on either end. + Streamed RPCs will generally need application-level pagination support, + because after a gRPC error one must retry the entire request; there is no + "retry-resume" functionality. + + Retries are handled with jittered exponential backoff to spread out failures + due to request spikes. + + Args: + api_method: Callable for the API method to invoke. + request: Request protocol buffer to pass to the API method. + clock: an interface object supporting `time()` and `sleep()` methods + like the standard `time` module; if not passed, uses the normal module. + + Returns: + Response protocol buffer returned by the API method. + + Raises: + grpc.RpcError: if a non-retryable error is returned, or if all retry + attempts have been exhausted. + """ + if clock is None: + clock = time + # We can't actually use api_method.__name__ because it's not a real method, + # it's a special gRPC callable instance that doesn't expose the method name. + rpc_name = request.__class__.__name__.replace("Request", "") + logger.debug("RPC call %s with request: %r", rpc_name, request) + num_attempts = 0 + while True: + num_attempts += 1 + try: + return api_method( + request, + timeout=_GRPC_DEFAULT_TIMEOUT_SECS, + metadata=version_metadata(), + ) + except grpc.RpcError as e: + logger.info("RPC call %s got error %s", rpc_name, e) + if e.code() not in _GRPC_RETRYABLE_STATUS_CODES: + raise + if num_attempts >= _GRPC_RETRY_MAX_ATTEMPTS: + raise + backoff_secs = _compute_backoff_seconds(num_attempts) + logger.info( + "RPC call %s attempted %d times, retrying in %.1f seconds", + rpc_name, + num_attempts, + backoff_secs, + ) + clock.sleep(backoff_secs) + + +def version_metadata(): + """Creates gRPC invocation metadata encoding the TensorBoard version. + + Usage: `stub.MyRpc(request, metadata=version_metadata())`. + + Returns: + A tuple of key-value pairs (themselves 2-tuples) to be passed as the + `metadata` kwarg to gRPC stub API methods. + """ + return ((_VERSION_METADATA_KEY, version.VERSION),) + + +def extract_version(metadata): + """Extracts version from invocation metadata. + + The argument should be the result of a prior call to `metadata` or the + result of combining such a result with other metadata. + + Returns: + The TensorBoard version listed in this metadata, or `None` if none + is listed. + """ + return dict(metadata).get(_VERSION_METADATA_KEY) + + +@enum.unique +class ChannelCredsType(enum.Enum): + LOCAL = "local" + SSL = "ssl" + SSL_DEV = "ssl_dev" + + def channel_config(self): + """Create channel credentials and options. + + Returns: + A tuple `(channel_creds, channel_options)`, where `channel_creds` + is a `grpc.ChannelCredentials` and `channel_options` is a + (potentially empty) list of `(key, value)` tuples. Both results + may be passed to `grpc.secure_channel`. + """ + + options = [] + if self == ChannelCredsType.LOCAL: + creds = grpc.local_channel_credentials() + elif self == ChannelCredsType.SSL: + creds = grpc.ssl_channel_credentials() + elif self == ChannelCredsType.SSL_DEV: + # Configure the dev cert to use by passing the environment variable + # GRPC_DEFAULT_SSL_ROOTS_FILE_PATH=path/to/cert.crt + creds = grpc.ssl_channel_credentials() + options.append(("grpc.ssl_target_name_override", "localhost")) + else: + raise AssertionError("unhandled ChannelCredsType: %r" % self) + return (creds, options) + + @classmethod + def choices(cls): + return cls.__members__.values() + + def __str__(self): + # Use user-facing string, because this is shown for flag choices. + return self.value diff --git a/lib/python3.12/site-packages/tensorboard/util/io_util.py b/lib/python3.12/site-packages/tensorboard/util/io_util.py new file mode 100644 index 0000000000000000000000000000000000000000..28294706a147c5bcf82a009b82ec62486922a6bc --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/io_util.py @@ -0,0 +1,20 @@ +# Copyright 2021 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TensorBoard IO helpers.""" + + +def IsCloudPath(path): + """Checks whether a given path is Cloud filesystem path.""" + return path.startswith("gs://") or path.startswith("s3://") diff --git a/lib/python3.12/site-packages/tensorboard/util/lazy_tensor_creator.py b/lib/python3.12/site-packages/tensorboard/util/lazy_tensor_creator.py new file mode 100644 index 0000000000000000000000000000000000000000..877dc2d7e56163fc30c17f0fe053594cc9126a2c --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/lazy_tensor_creator.py @@ -0,0 +1,111 @@ +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Provides a lazy wrapper for deferring Tensor creation.""" + +import threading + +from tensorboard.compat import tf2 as tf + + +# Sentinel used for LazyTensorCreator._tensor to indicate that a value is +# currently being computed, in order to fail hard on reentrancy. +_CALL_IN_PROGRESS_SENTINEL = object() + + +class LazyTensorCreator: + """Lazy auto-converting wrapper for a callable that returns a `tf.Tensor`. + + This class wraps an arbitrary callable that returns a `Tensor` so that it + will be automatically converted to a `Tensor` by any logic that calls + `tf.convert_to_tensor()`. This also memoizes the callable so that it is + called at most once. + + The intended use of this class is to defer the construction of a `Tensor` + (e.g. to avoid unnecessary wasted computation, or ensure any new ops are + created in a context only available later on in execution), while remaining + compatible with APIs that expect to be given an already materialized value + that can be converted to a `Tensor`. + + This class is thread-safe. + """ + + def __init__(self, tensor_callable): + """Initializes a LazyTensorCreator object. + + Args: + tensor_callable: A callable that returns a `tf.Tensor`. + """ + if not callable(tensor_callable): + raise ValueError("Not a callable: %r" % tensor_callable) + self._tensor_callable = tensor_callable + self._tensor = None + self._tensor_lock = threading.RLock() + _register_conversion_function_once() + + def __call__(self): + if self._tensor is None or self._tensor is _CALL_IN_PROGRESS_SENTINEL: + with self._tensor_lock: + if self._tensor is _CALL_IN_PROGRESS_SENTINEL: + raise RuntimeError( + "Cannot use LazyTensorCreator with reentrant callable" + ) + elif self._tensor is None: + self._tensor = _CALL_IN_PROGRESS_SENTINEL + self._tensor = self._tensor_callable() + return self._tensor + + +def _lazy_tensor_creator_converter(value, dtype=None, name=None, as_ref=False): + del name # ignored + if not isinstance(value, LazyTensorCreator): + raise RuntimeError("Expected LazyTensorCreator, got %r" % value) + if as_ref: + raise RuntimeError("Cannot use LazyTensorCreator to create ref tensor") + tensor = value() + if dtype not in (None, tensor.dtype): + raise RuntimeError( + "Cannot convert LazyTensorCreator returning dtype %s to dtype %s" + % (tensor.dtype, dtype) + ) + return tensor + + +# Use module-level bit and lock to ensure that registration of the +# LazyTensorCreator conversion function happens only once. +_conversion_registered = False +_conversion_registered_lock = threading.Lock() + + +def _register_conversion_function_once(): + """Performs one-time registration of `_lazy_tensor_creator_converter`. + + This helper can be invoked multiple times but only registers the conversion + function on the first invocation, making it suitable for calling when + constructing a LazyTensorCreator. + + Deferring the registration is necessary because doing it at at module import + time would trigger the lazy TensorFlow import to resolve, and that in turn + would break the delicate `tf.summary` import cycle avoidance scheme. + """ + global _conversion_registered + if not _conversion_registered: + with _conversion_registered_lock: + if not _conversion_registered: + _conversion_registered = True + tf.register_tensor_conversion_function( + base_type=LazyTensorCreator, + conversion_func=_lazy_tensor_creator_converter, + priority=0, + ) diff --git a/lib/python3.12/site-packages/tensorboard/util/op_evaluator.py b/lib/python3.12/site-packages/tensorboard/util/op_evaluator.py new file mode 100644 index 0000000000000000000000000000000000000000..f42a0653a1c0a1da73788e1ad8724a32e11ec1a4 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/op_evaluator.py @@ -0,0 +1,105 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TensorBoard helper routine for TF op evaluator. + +Requires TensorFlow. +""" + + +import threading + + +class PersistentOpEvaluator: + """Evaluate a fixed TensorFlow graph repeatedly, safely, efficiently. + + Extend this class to create a particular kind of op evaluator, like an + image encoder. In `initialize_graph`, create an appropriate TensorFlow + graph with placeholder inputs. In `run`, evaluate this graph and + return its result. This class will manage a singleton graph and + session to preserve memory usage, and will ensure that this graph and + session do not interfere with other concurrent sessions. + + A subclass of this class offers a threadsafe, highly parallel Python + entry point for evaluating a particular TensorFlow graph. + + Example usage: + + class FluxCapacitanceEvaluator(PersistentOpEvaluator): + \"\"\"Compute the flux capacitance required for a system. + + Arguments: + x: Available power input, as a `float`, in jigawatts. + + Returns: + A `float`, in nanofarads. + \"\"\" + + def initialize_graph(self): + self._placeholder = tf.placeholder(some_dtype) + self._op = some_op(self._placeholder) + + def run(self, x): + return self._op.eval(feed_dict: {self._placeholder: x}) + + evaluate_flux_capacitance = FluxCapacitanceEvaluator() + + for x in xs: + evaluate_flux_capacitance(x) + """ + + def __init__(self): + super().__init__() + self._session = None + self._initialization_lock = threading.Lock() + + def _lazily_initialize(self): + """Initialize the graph and session, if this has not yet been done.""" + # TODO(nickfelt): remove on-demand imports once dep situation is fixed. + import tensorflow.compat.v1 as tf + + with self._initialization_lock: + if self._session: + return + graph = tf.Graph() + with graph.as_default(): + self.initialize_graph() + # Don't reserve GPU because libpng can't run on GPU. + config = tf.ConfigProto(device_count={"GPU": 0}) + self._session = tf.Session(graph=graph, config=config) + + def initialize_graph(self): + """Create the TensorFlow graph needed to compute this operation. + + This should write ops to the default graph and return `None`. + """ + raise NotImplementedError( + 'Subclasses must implement "initialize_graph".' + ) + + def run(self, *args, **kwargs): + """Evaluate the ops with the given input. + + When this function is called, the default session will have the + graph defined by a previous call to `initialize_graph`. This + function should evaluate any ops necessary to compute the result + of the query for the given *args and **kwargs, likely returning + the result of a call to `some_op.eval(...)`. + """ + raise NotImplementedError('Subclasses must implement "run".') + + def __call__(self, *args, **kwargs): + self._lazily_initialize() + with self._session.as_default(): + return self.run(*args, **kwargs) diff --git a/lib/python3.12/site-packages/tensorboard/util/platform_util.py b/lib/python3.12/site-packages/tensorboard/util/platform_util.py new file mode 100644 index 0000000000000000000000000000000000000000..274a1503123c806f4c48a4157f346faf336284b2 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/platform_util.py @@ -0,0 +1,20 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TensorBoard helper routine for platform.""" + + +def readahead_file_path(path, unused_readahead=None): + """Readahead files not implemented; simply returns given path.""" + return path diff --git a/lib/python3.12/site-packages/tensorboard/util/tb_logging.py b/lib/python3.12/site-packages/tensorboard/util/tb_logging.py new file mode 100644 index 0000000000000000000000000000000000000000..f49a5b2a204f19a5b1aaf265d60ea8728633f6b7 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/tb_logging.py @@ -0,0 +1,25 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""TensorBoard logging module.""" + +import logging + + +_logger = logging.getLogger("tensorboard") + + +def get_logger(): + """Returns TensorBoard logger.""" + return _logger diff --git a/lib/python3.12/site-packages/tensorboard/util/tensor_util.py b/lib/python3.12/site-packages/tensorboard/util/tensor_util.py new file mode 100644 index 0000000000000000000000000000000000000000..cbf4758925ee3cd3c38a7d125d463a08f2f5a2b8 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/tensor_util.py @@ -0,0 +1,617 @@ +# Copyright 2018 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities to manipulate TensorProtos.""" + +import numpy as np + +from tensorboard.compat.proto import tensor_pb2 +from tensorboard.compat.tensorflow_stub import dtypes, compat, tensor_shape + + +def ExtractBitsFromFloat16(x): + return np.asarray(x, dtype=np.float16).view(np.uint16).item() + + +def SlowAppendFloat16ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.half_val.extend( + [ExtractBitsFromFloat16(x) for x in proto_values] + ) + + +def ExtractBitsFromBFloat16(x): + return ( + np.asarray(x, dtype=dtypes.bfloat16.as_numpy_dtype) + .view(np.uint16) + .item() + ) + + +def SlowAppendBFloat16ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.half_val.extend( + [ExtractBitsFromBFloat16(x) for x in proto_values] + ) + + +def SlowAppendFloat32ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.float_val.extend([x.item() for x in proto_values]) + + +def SlowAppendFloat64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.double_val.extend([x.item() for x in proto_values]) + + +def SlowAppendIntArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.int_val.extend([x.item() for x in proto_values]) + + +def SlowAppendInt64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.int64_val.extend([x.item() for x in proto_values]) + + +def SlowAppendQIntArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.int_val.extend([x[0].item() for x in proto_values]) + + +def SlowAppendUInt32ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.uint32_val.extend([x.item() for x in proto_values]) + + +def SlowAppendUInt64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.uint64_val.extend([x.item() for x in proto_values]) + + +def SlowAppendComplex64ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.scomplex_val.extend( + [v.item() for x in proto_values for v in [x.real, x.imag]] + ) + + +def SlowAppendComplex128ArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.dcomplex_val.extend( + [v.item() for x in proto_values for v in [x.real, x.imag]] + ) + + +def SlowAppendObjectArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.string_val.extend([compat.as_bytes(x) for x in proto_values]) + + +def SlowAppendBoolArrayToTensorProto(tensor_proto, proto_values): + tensor_proto.bool_val.extend([x.item() for x in proto_values]) + + +_NP_TO_APPEND_FN = { + np.float16: SlowAppendFloat16ArrayToTensorProto, + np.float32: SlowAppendFloat32ArrayToTensorProto, + np.float64: SlowAppendFloat64ArrayToTensorProto, + np.int32: SlowAppendIntArrayToTensorProto, + np.int64: SlowAppendInt64ArrayToTensorProto, + np.uint8: SlowAppendIntArrayToTensorProto, + np.uint16: SlowAppendIntArrayToTensorProto, + np.uint32: SlowAppendUInt32ArrayToTensorProto, + np.uint64: SlowAppendUInt64ArrayToTensorProto, + np.int8: SlowAppendIntArrayToTensorProto, + np.int16: SlowAppendIntArrayToTensorProto, + np.complex64: SlowAppendComplex64ArrayToTensorProto, + np.complex128: SlowAppendComplex128ArrayToTensorProto, + np.object_: SlowAppendObjectArrayToTensorProto, + np.bool_: SlowAppendBoolArrayToTensorProto, + dtypes.qint8.as_numpy_dtype: SlowAppendQIntArrayToTensorProto, + dtypes.quint8.as_numpy_dtype: SlowAppendQIntArrayToTensorProto, + dtypes.qint16.as_numpy_dtype: SlowAppendQIntArrayToTensorProto, + dtypes.quint16.as_numpy_dtype: SlowAppendQIntArrayToTensorProto, + dtypes.qint32.as_numpy_dtype: SlowAppendQIntArrayToTensorProto, + # NOTE(touts): Intentionally no way to feed a DT_BFLOAT16. +} + +BACKUP_DICT = { + dtypes.bfloat16.as_numpy_dtype: SlowAppendBFloat16ArrayToTensorProto +} + + +def GetFromNumpyDTypeDict(dtype_dict, dtype): + # NOTE: dtype_dict.get(dtype) always returns None. + for key, val in dtype_dict.items(): + if key == dtype: + return val + for key, val in BACKUP_DICT.items(): + if key == dtype: + return val + return None + + +def GetNumpyAppendFn(dtype): + # numpy dtype for strings are variable length. We can not compare + # dtype with a single constant (np.string does not exist) to decide + # dtype is a "string" type. We need to compare the dtype.type to be + # sure it's a string type. + if dtype.type == np.bytes_ or dtype.type == np.str_: + return SlowAppendObjectArrayToTensorProto + return GetFromNumpyDTypeDict(_NP_TO_APPEND_FN, dtype) + + +def _GetDenseDimensions(list_of_lists): + """Returns the inferred dense dimensions of a list of lists.""" + if not isinstance(list_of_lists, (list, tuple)): + return [] + elif not list_of_lists: + return [0] + else: + return [len(list_of_lists)] + _GetDenseDimensions(list_of_lists[0]) + + +def _FlattenToStrings(nested_strings): + if isinstance(nested_strings, (list, tuple)): + for inner in nested_strings: + for flattened_string in _FlattenToStrings(inner): + yield flattened_string + else: + yield nested_strings + + +_TENSOR_CONTENT_TYPES = frozenset( + [ + dtypes.float32, + dtypes.float64, + dtypes.int32, + dtypes.uint8, + dtypes.int16, + dtypes.int8, + dtypes.int64, + dtypes.qint8, + dtypes.quint8, + dtypes.qint16, + dtypes.quint16, + dtypes.qint32, + dtypes.uint32, + dtypes.uint64, + ] +) + + +class _Message: + def __init__(self, message): + self._message = message + + def __repr__(self): + return self._message + + +def _FirstNotNone(l): + for x in l: + if x is not None: + return x + return None + + +def _NotNone(v): + if v is None: + return _Message("None") + else: + return v + + +def _FilterTuple(v): + if not isinstance(v, (list, tuple)): + return v + if isinstance(v, tuple): + if not any(isinstance(x, (list, tuple)) for x in v): + return None + if isinstance(v, list): + if not any(isinstance(x, (list, tuple)) for x in v): + return _FirstNotNone( + [None if isinstance(x, (list, tuple)) else x for x in v] + ) + return _FirstNotNone([_FilterTuple(x) for x in v]) + + +def _FilterInt(v): + if isinstance(v, (list, tuple)): + return _FirstNotNone([_FilterInt(x) for x in v]) + return ( + None + if isinstance(v, (compat.integral_types, tensor_shape.Dimension)) + else _NotNone(v) + ) + + +def _FilterFloat(v): + if isinstance(v, (list, tuple)): + return _FirstNotNone([_FilterFloat(x) for x in v]) + return None if isinstance(v, compat.real_types) else _NotNone(v) + + +def _FilterComplex(v): + if isinstance(v, (list, tuple)): + return _FirstNotNone([_FilterComplex(x) for x in v]) + return None if isinstance(v, compat.complex_types) else _NotNone(v) + + +def _FilterStr(v): + if isinstance(v, (list, tuple)): + return _FirstNotNone([_FilterStr(x) for x in v]) + if isinstance(v, compat.bytes_or_text_types): + return None + else: + return _NotNone(v) + + +def _FilterBool(v): + if isinstance(v, (list, tuple)): + return _FirstNotNone([_FilterBool(x) for x in v]) + return None if isinstance(v, bool) else _NotNone(v) + + +_TF_TO_IS_OK = { + dtypes.bool: [_FilterBool], + dtypes.complex128: [_FilterComplex], + dtypes.complex64: [_FilterComplex], + dtypes.float16: [_FilterFloat], + dtypes.float32: [_FilterFloat], + dtypes.float64: [_FilterFloat], + dtypes.int16: [_FilterInt], + dtypes.int32: [_FilterInt], + dtypes.int64: [_FilterInt], + dtypes.int8: [_FilterInt], + dtypes.qint16: [_FilterInt, _FilterTuple], + dtypes.qint32: [_FilterInt, _FilterTuple], + dtypes.qint8: [_FilterInt, _FilterTuple], + dtypes.quint16: [_FilterInt, _FilterTuple], + dtypes.quint8: [_FilterInt, _FilterTuple], + dtypes.string: [_FilterStr], + dtypes.uint16: [_FilterInt], + dtypes.uint8: [_FilterInt], +} + + +def _Assertconvertible(values, dtype): + # If dtype is None or not recognized, assume it's convertible. + if dtype is None or dtype not in _TF_TO_IS_OK: + return + fn_list = _TF_TO_IS_OK.get(dtype) + mismatch = _FirstNotNone([fn(values) for fn in fn_list]) + if mismatch is not None: + raise TypeError( + "Expected %s, got %s of type '%s' instead." + % (dtype.name, repr(mismatch), type(mismatch).__name__) + ) + + +def make_tensor_proto(values, dtype=None, shape=None, verify_shape=False): + """Create a TensorProto. + + Args: + values: Values to put in the TensorProto. + dtype: Optional tensor_pb2 DataType value. + shape: List of integers representing the dimensions of tensor. + verify_shape: Boolean that enables verification of a shape of values. + + Returns: + A `TensorProto`. Depending on the type, it may contain data in the + "tensor_content" attribute, which is not directly useful to Python programs. + To access the values you should convert the proto back to a numpy ndarray + with `tensor_util.MakeNdarray(proto)`. + + If `values` is a `TensorProto`, it is immediately returned; `dtype` and + `shape` are ignored. + + Raises: + TypeError: if unsupported types are provided. + ValueError: if arguments have inappropriate values or if verify_shape is + True and shape of values is not equals to a shape from the argument. + + make_tensor_proto accepts "values" of a python scalar, a python list, a + numpy ndarray, or a numpy scalar. + + If "values" is a python scalar or a python list, make_tensor_proto + first convert it to numpy ndarray. If dtype is None, the + conversion tries its best to infer the right numpy data + type. Otherwise, the resulting numpy array has a convertible data + type with the given dtype. + + In either case above, the numpy ndarray (either the caller provided + or the auto converted) must have the convertible type with dtype. + + make_tensor_proto then converts the numpy array to a tensor proto. + + If "shape" is None, the resulting tensor proto represents the numpy + array precisely. + + Otherwise, "shape" specifies the tensor's shape and the numpy array + can not have more elements than what "shape" specifies. + """ + if isinstance(values, tensor_pb2.TensorProto): + return values + + if dtype: + dtype = dtypes.as_dtype(dtype) + + is_quantized = dtype in [ + dtypes.qint8, + dtypes.quint8, + dtypes.qint16, + dtypes.quint16, + dtypes.qint32, + ] + + # We first convert value to a numpy array or scalar. + if isinstance(values, (np.ndarray, np.generic)): + if dtype: + nparray = values.astype(dtype.as_numpy_dtype) + else: + nparray = values + elif callable(getattr(values, "__array__", None)) or isinstance( + getattr(values, "__array_interface__", None), dict + ): + # If a class has the __array__ method, or __array_interface__ dict, then it + # is possible to convert to numpy array. + nparray = np.asarray(values, dtype=dtype) + + # This is the preferred way to create an array from the object, so replace + # the `values` with the array so that _FlattenToStrings is not run. + values = nparray + else: + if values is None: + raise ValueError("None values not supported.") + # if dtype is provided, forces numpy array to be the type + # provided if possible. + if dtype and dtype.is_numpy_compatible: + np_dt = dtype.as_numpy_dtype + else: + np_dt = None + # If shape is None, numpy.prod returns None when dtype is not set, but raises + # exception when dtype is set to np.int64 + if shape is not None and np.prod(shape, dtype=np.int64) == 0: + nparray = np.empty(shape, dtype=np_dt) + else: + _Assertconvertible(values, dtype) + nparray = np.array(values, dtype=np_dt) + # check to them. + # We need to pass in quantized values as tuples, so don't apply the shape + if ( + list(nparray.shape) != _GetDenseDimensions(values) + and not is_quantized + ): + raise ValueError( + """Argument must be a dense tensor: %s""" + """ - got shape %s, but wanted %s.""" + % (values, list(nparray.shape), _GetDenseDimensions(values)) + ) + + # python/numpy default float type is float64. We prefer float32 instead. + if (nparray.dtype == np.float64) and dtype is None: + nparray = nparray.astype(np.float32) + # python/numpy default int type is int64. We prefer int32 instead. + elif (nparray.dtype == np.int64) and dtype is None: + downcasted_array = nparray.astype(np.int32) + # Do not down cast if it leads to precision loss. + if np.array_equal(downcasted_array, nparray): + nparray = downcasted_array + + # if dtype is provided, it must be convertible with what numpy + # conversion says. + numpy_dtype = dtypes.as_dtype(nparray.dtype) + if numpy_dtype is None: + raise TypeError("Unrecognized data type: %s" % nparray.dtype) + + # If dtype was specified and is a quantized type, we convert + # numpy_dtype back into the quantized version. + if is_quantized: + numpy_dtype = dtype + + if dtype is not None and ( + not hasattr(dtype, "base_dtype") + or dtype.base_dtype != numpy_dtype.base_dtype + ): + raise TypeError( + "Inconvertible types: %s vs. %s. Value is %s" + % (dtype, nparray.dtype, values) + ) + + # If shape is not given, get the shape from the numpy array. + if shape is None: + shape = nparray.shape + is_same_size = True + shape_size = nparray.size + else: + shape = [int(dim) for dim in shape] + shape_size = np.prod(shape, dtype=np.int64) + is_same_size = shape_size == nparray.size + + if verify_shape: + if not nparray.shape == tuple(shape): + raise TypeError( + "Expected Tensor's shape: %s, got %s." + % (tuple(shape), nparray.shape) + ) + + if nparray.size > shape_size: + raise ValueError( + "Too many elements provided. Needed at most %d, but received %d" + % (shape_size, nparray.size) + ) + + tensor_proto = tensor_pb2.TensorProto( + dtype=numpy_dtype.as_datatype_enum, + tensor_shape=tensor_shape.as_shape(shape).as_proto(), + ) + + if is_same_size and numpy_dtype in _TENSOR_CONTENT_TYPES and shape_size > 1: + if nparray.size * nparray.itemsize >= (1 << 31): + raise ValueError( + "Cannot create a tensor proto whose content is larger than 2GB." + ) + tensor_proto.tensor_content = nparray.tobytes() + return tensor_proto + + # If we were not given values as a numpy array, compute the proto_values + # from the given values directly, to avoid numpy trimming nulls from the + # strings. Since values could be a list of strings, or a multi-dimensional + # list of lists that might or might not correspond to the given shape, + # we flatten it conservatively. + if numpy_dtype == dtypes.string and not isinstance(values, np.ndarray): + proto_values = _FlattenToStrings(values) + + # At this point, values may be a list of objects that we could not + # identify a common type for (hence it was inferred as + # np.object/dtypes.string). If we are unable to convert it to a + # string, we raise a more helpful error message. + # + # Ideally, we'd be able to convert the elements of the list to a + # common type, but this type inference requires some thinking and + # so we defer it for now. + try: + str_values = [compat.as_bytes(x) for x in proto_values] + except TypeError: + raise TypeError( + "Failed to convert object of type %s to Tensor. " + "Contents: %s. Consider casting elements to a " + "supported type." % (type(values), values) + ) + tensor_proto.string_val.extend(str_values) + return tensor_proto + + # TensorFlow expects C order (a.k.a., eigen row major). + proto_values = nparray.ravel() + + append_fn = GetNumpyAppendFn(proto_values.dtype) + if append_fn is None: + raise TypeError( + "Element type not supported in TensorProto: %s" % numpy_dtype.name + ) + append_fn(tensor_proto, proto_values) + + return tensor_proto + + +def make_ndarray(tensor): + """Create a numpy ndarray from a tensor. + + Create a numpy ndarray with the same shape and data as the tensor. + + Args: + tensor: A TensorProto. + + Returns: + A numpy array with the tensor contents. + + Raises: + TypeError: if tensor has unsupported type. + """ + shape = [d.size for d in tensor.tensor_shape.dim] + num_elements = np.prod(shape, dtype=np.int64) + tensor_dtype = dtypes.as_dtype(tensor.dtype) + dtype = tensor_dtype.as_numpy_dtype + + if tensor.tensor_content: + return ( + np.frombuffer(tensor.tensor_content, dtype=dtype) + .copy() + .reshape(shape) + ) + elif tensor_dtype == dtypes.float16 or tensor_dtype == dtypes.bfloat16: + # the half_val field of the TensorProto stores the binary representation + # of the fp16: we need to reinterpret this as a proper float16 + if len(tensor.half_val) == 1: + tmp = np.array(tensor.half_val[0], dtype=np.uint16) + tmp.dtype = tensor_dtype.as_numpy_dtype + return np.repeat(tmp, num_elements).reshape(shape) + else: + tmp = np.fromiter(tensor.half_val, dtype=np.uint16) + tmp.dtype = tensor_dtype.as_numpy_dtype + return tmp.reshape(shape) + elif tensor_dtype == dtypes.float32: + if len(tensor.float_val) == 1: + return np.repeat( + np.array(tensor.float_val[0], dtype=dtype), num_elements + ).reshape(shape) + else: + return np.fromiter(tensor.float_val, dtype=dtype).reshape(shape) + elif tensor_dtype == dtypes.float64: + if len(tensor.double_val) == 1: + return np.repeat( + np.array(tensor.double_val[0], dtype=dtype), num_elements + ).reshape(shape) + else: + return np.fromiter(tensor.double_val, dtype=dtype).reshape(shape) + elif tensor_dtype in [ + dtypes.int32, + dtypes.uint8, + dtypes.uint16, + dtypes.int16, + dtypes.int8, + dtypes.qint32, + dtypes.quint8, + dtypes.qint8, + dtypes.qint16, + dtypes.quint16, + ]: + if len(tensor.int_val) == 1: + return np.repeat( + np.array(tensor.int_val[0], dtype=dtype), num_elements + ).reshape(shape) + else: + return np.fromiter(tensor.int_val, dtype=dtype).reshape(shape) + elif tensor_dtype == dtypes.int64: + if len(tensor.int64_val) == 1: + return np.repeat( + np.array(tensor.int64_val[0], dtype=dtype), num_elements + ).reshape(shape) + else: + return np.fromiter(tensor.int64_val, dtype=dtype).reshape(shape) + elif tensor_dtype == dtypes.string: + if len(tensor.string_val) == 1: + return np.repeat( + np.array(tensor.string_val[0], dtype=dtype), num_elements + ).reshape(shape) + else: + return np.array(list(tensor.string_val), dtype=dtype).reshape(shape) + elif tensor_dtype == dtypes.complex64: + it = iter(tensor.scomplex_val) + if len(tensor.scomplex_val) == 2: + return np.repeat( + np.array( + complex(tensor.scomplex_val[0], tensor.scomplex_val[1]), + dtype=dtype, + ), + num_elements, + ).reshape(shape) + else: + return np.array( + [complex(x[0], x[1]) for x in zip(it, it)], dtype=dtype + ).reshape(shape) + elif tensor_dtype == dtypes.complex128: + it = iter(tensor.dcomplex_val) + if len(tensor.dcomplex_val) == 2: + return np.repeat( + np.array( + complex(tensor.dcomplex_val[0], tensor.dcomplex_val[1]), + dtype=dtype, + ), + num_elements, + ).reshape(shape) + else: + return np.array( + [complex(x[0], x[1]) for x in zip(it, it)], dtype=dtype + ).reshape(shape) + elif tensor_dtype == dtypes.bool: + if len(tensor.bool_val) == 1: + return np.repeat( + np.array(tensor.bool_val[0], dtype=dtype), num_elements + ).reshape(shape) + else: + return np.fromiter(tensor.bool_val, dtype=dtype).reshape(shape) + else: + raise TypeError("Unsupported tensor type: %s" % tensor.dtype) diff --git a/lib/python3.12/site-packages/tensorboard/util/timing.py b/lib/python3.12/site-packages/tensorboard/util/timing.py new file mode 100644 index 0000000000000000000000000000000000000000..8353eaa3cbedb56a9a378b9e2eb3643a557f6b70 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/util/timing.py @@ -0,0 +1,122 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Utilities for measuring elapsed time.""" + +import contextlib +import logging +import threading +import time + +from tensorboard.util import tb_logging + +logger = tb_logging.get_logger() + + +def log_latency(region_name_or_function_to_decorate, log_level=None): + """Log latency in a function or region. + + Three usages are supported. As a decorator: + + >>> @log_latency + ... def function_1(): + ... pass + ... + + + As a decorator with a custom label for the region: + + >>> @log_latency("custom_label") + ... def function_2(): + ... pass + ... + + As a context manager: + + >>> def function_3(): + ... with log_latency("region_within_function"): + ... pass + ... + + Args: + region_name_or_function_to_decorate: Either: a `str`, in which + case the result of this function may be used as either a + decorator or a context manager; or a callable, in which case + the result of this function is a decorated version of that + callable. + log_level: Optional integer logging level constant. Defaults to + `logging.INFO`. + + Returns: + A decorated version of the input callable, or a dual + decorator/context manager with the input region name. + """ + + if log_level is None: + log_level = logging.INFO + + if isinstance(region_name_or_function_to_decorate, str): + region_name = region_name_or_function_to_decorate + return _log_latency(region_name, log_level) + else: + function_to_decorate = region_name_or_function_to_decorate + qualname = getattr(function_to_decorate, "__qualname__", None) + if qualname is None: + qualname = str(function_to_decorate) + decorator = _log_latency(qualname, log_level) + return decorator(function_to_decorate) + + +class _ThreadLocalStore(threading.local): + def __init__(self): + self.nesting_level = 0 + + +_store = _ThreadLocalStore() + + +@contextlib.contextmanager +def _log_latency(name, log_level): + if not logger.isEnabledFor(log_level): + yield + return + + start_level = _store.nesting_level + try: + started = time.time() + _store.nesting_level = start_level + 1 + indent = (" " * 2) * start_level + thread = threading.current_thread() + prefix = "%s[%x]%s" % (thread.name, thread.ident, indent) + _log(log_level, "%s ENTER %s", prefix, name) + yield + finally: + _store.nesting_level = start_level + elapsed = time.time() - started + _log( + log_level, + "%s LEAVE %s - %0.6fs elapsed", + prefix, + name, + elapsed, + ) + + +def _log(log_level, msg, *args): + # Forwarding method to ensure that all logging statements + # originating in this module have the same line number; if the + # "ENTER" log is on a line with 2-digit number and the "LEAVE" log + # is on a line with 3-digit number, the logs are misaligned and + # harder to read. + logger.log(log_level, msg, *args) diff --git a/lib/python3.12/site-packages/tensorboard/version.py b/lib/python3.12/site-packages/tensorboard/version.py new file mode 100644 index 0000000000000000000000000000000000000000..ae6a803824c437468c5f4a0fd483d01c2d929ae9 --- /dev/null +++ b/lib/python3.12/site-packages/tensorboard/version.py @@ -0,0 +1,21 @@ +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +"""Contains the version string.""" + +VERSION = "2.19.0" + +if __name__ == "__main__": + print(VERSION) diff --git a/lib/python3.12/site-packages/werkzeug/__init__.py b/lib/python3.12/site-packages/werkzeug/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b248fd86152e58889a226aa628c6222fe23ece1 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/__init__.py @@ -0,0 +1,4 @@ +from .serving import run_simple as run_simple +from .test import Client as Client +from .wrappers import Request as Request +from .wrappers import Response as Response diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..af7a5cfef3681cccf57e08568d38e5c2fc7bc6f4 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/_internal.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/_internal.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49badba62374760e2a63a989dc93df06f4ba4db4 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/_internal.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/_reloader.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/_reloader.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d7b2fe8db07aa742a7e02d4e9267b305605c1d2d Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/_reloader.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/exceptions.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1191d32b73ff9c92cd821c315f09c00e0a37da2 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/exceptions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/formparser.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/formparser.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3870fd81950bc115cdc53e0039671c7f17df3976 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/formparser.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/http.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/http.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d5a6c66d10856f284d2d7626b5a502707b3c831 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/http.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/local.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/local.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0d8fb09767fc6550ac247045805e3fc4b74f5230 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/local.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/security.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/security.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..63c4b4b2283e2fdf8f07ad2dda2cb03b55186b9e Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/security.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/serving.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/serving.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f141f6f4023b34d59de79fb4440461cd2b1675ce Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/serving.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/test.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/test.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e97a3ceb706f7bc44154d39716e2115cabb62e92 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/test.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/testapp.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/testapp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e4f11abf45f1bb863bfab4a0a8dade6ecadd0b44 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/testapp.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/urls.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/urls.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb8ffe78e2252fc4b528978c31e19704658c0406 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/urls.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/user_agent.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/user_agent.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa496a0ea5bb965d0817c86c2c180ce13b23f8c9 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/user_agent.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/utils.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8157fc886bccc0109b6f8044be7f7c35b7d126ea Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/__pycache__/wsgi.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/__pycache__/wsgi.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..67e19c356188db222b0ae56bdfadbc8292b4db2f Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/__pycache__/wsgi.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/_internal.py b/lib/python3.12/site-packages/werkzeug/_internal.py new file mode 100644 index 0000000000000000000000000000000000000000..75e43358b629ec88692ca87d57deddc2252c554a --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/_internal.py @@ -0,0 +1,211 @@ +from __future__ import annotations + +import logging +import re +import sys +import typing as t +from datetime import datetime +from datetime import timezone + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIEnvironment + + from .wrappers.request import Request + +_logger: logging.Logger | None = None + + +class _Missing: + def __repr__(self) -> str: + return "no value" + + def __reduce__(self) -> str: + return "_missing" + + +_missing = _Missing() + + +def _wsgi_decoding_dance(s: str) -> str: + return s.encode("latin1").decode(errors="replace") + + +def _wsgi_encoding_dance(s: str) -> str: + return s.encode().decode("latin1") + + +def _get_environ(obj: WSGIEnvironment | Request) -> WSGIEnvironment: + env = getattr(obj, "environ", obj) + assert isinstance(env, dict), ( + f"{type(obj).__name__!r} is not a WSGI environment (has to be a dict)" + ) + return env + + +def _has_level_handler(logger: logging.Logger) -> bool: + """Check if there is a handler in the logging chain that will handle + the given logger's effective level. + """ + level = logger.getEffectiveLevel() + current = logger + + while current: + if any(handler.level <= level for handler in current.handlers): + return True + + if not current.propagate: + break + + current = current.parent # type: ignore + + return False + + +class _ColorStreamHandler(logging.StreamHandler): # type: ignore[type-arg] + """On Windows, wrap stream with Colorama for ANSI style support.""" + + def __init__(self) -> None: + try: + import colorama + except ImportError: + stream = None + else: + stream = colorama.AnsiToWin32(sys.stderr) + + super().__init__(stream) + + +def _log(type: str, message: str, *args: t.Any, **kwargs: t.Any) -> None: + """Log a message to the 'werkzeug' logger. + + The logger is created the first time it is needed. If there is no + level set, it is set to :data:`logging.INFO`. If there is no handler + for the logger's effective level, a :class:`logging.StreamHandler` + is added. + """ + global _logger + + if _logger is None: + _logger = logging.getLogger("werkzeug") + + if _logger.level == logging.NOTSET: + _logger.setLevel(logging.INFO) + + if not _has_level_handler(_logger): + _logger.addHandler(_ColorStreamHandler()) + + getattr(_logger, type)(message.rstrip(), *args, **kwargs) + + +@t.overload +def _dt_as_utc(dt: None) -> None: ... + + +@t.overload +def _dt_as_utc(dt: datetime) -> datetime: ... + + +def _dt_as_utc(dt: datetime | None) -> datetime | None: + if dt is None: + return dt + + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + elif dt.tzinfo != timezone.utc: + return dt.astimezone(timezone.utc) + + return dt + + +_TAccessorValue = t.TypeVar("_TAccessorValue") + + +class _DictAccessorProperty(t.Generic[_TAccessorValue]): + """Baseclass for `environ_property` and `header_property`.""" + + read_only = False + + def __init__( + self, + name: str, + default: _TAccessorValue | None = None, + load_func: t.Callable[[str], _TAccessorValue] | None = None, + dump_func: t.Callable[[_TAccessorValue], str] | None = None, + read_only: bool | None = None, + doc: str | None = None, + ) -> None: + self.name = name + self.default = default + self.load_func = load_func + self.dump_func = dump_func + if read_only is not None: + self.read_only = read_only + self.__doc__ = doc + + def lookup(self, instance: t.Any) -> t.MutableMapping[str, t.Any]: + raise NotImplementedError + + @t.overload + def __get__( + self, instance: None, owner: type + ) -> _DictAccessorProperty[_TAccessorValue]: ... + + @t.overload + def __get__(self, instance: t.Any, owner: type) -> _TAccessorValue: ... + + def __get__( + self, instance: t.Any | None, owner: type + ) -> _TAccessorValue | _DictAccessorProperty[_TAccessorValue]: + if instance is None: + return self + + storage = self.lookup(instance) + + if self.name not in storage: + return self.default # type: ignore + + value = storage[self.name] + + if self.load_func is not None: + try: + return self.load_func(value) + except (ValueError, TypeError): + return self.default # type: ignore + + return value # type: ignore + + def __set__(self, instance: t.Any, value: _TAccessorValue) -> None: + if self.read_only: + raise AttributeError("read only property") + + if self.dump_func is not None: + self.lookup(instance)[self.name] = self.dump_func(value) + else: + self.lookup(instance)[self.name] = value + + def __delete__(self, instance: t.Any) -> None: + if self.read_only: + raise AttributeError("read only property") + + self.lookup(instance).pop(self.name, None) + + def __repr__(self) -> str: + return f"<{type(self).__name__} {self.name}>" + + +_plain_int_re = re.compile(r"-?\d+", re.ASCII) + + +def _plain_int(value: str) -> int: + """Parse an int only if it is only ASCII digits and ``-``. + + This disallows ``+``, ``_``, and non-ASCII digits, which are accepted by ``int`` but + are not allowed in HTTP header values. + + Any leading or trailing whitespace is stripped + """ + value = value.strip() + if _plain_int_re.fullmatch(value) is None: + raise ValueError + + return int(value) diff --git a/lib/python3.12/site-packages/werkzeug/_reloader.py b/lib/python3.12/site-packages/werkzeug/_reloader.py new file mode 100644 index 0000000000000000000000000000000000000000..5eb64bc7d0db4acd88f10bd36bdf57822c2f5df3 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/_reloader.py @@ -0,0 +1,465 @@ +from __future__ import annotations + +import fnmatch +import os +import subprocess +import sys +import threading +import time +import typing as t +from itertools import chain +from pathlib import PurePath + +from ._internal import _log + +# The various system prefixes where imports are found. Base values are +# different when running in a virtualenv. All reloaders will ignore the +# base paths (usually the system installation). The stat reloader won't +# scan the virtualenv paths, it will only include modules that are +# already imported. +_ignore_always = tuple({sys.base_prefix, sys.base_exec_prefix}) +prefix = {*_ignore_always, sys.prefix, sys.exec_prefix} + +if hasattr(sys, "real_prefix"): + # virtualenv < 20 + prefix.add(sys.real_prefix) + +_stat_ignore_scan = tuple(prefix) +del prefix +# Ignore __pycache__ since a change there will always have a change to +# the source file (or initial pyc file) as well. Ignore common version control +# internals. Ignore common tool caches. +_ignore_common_dirs = { + "__pycache__", + ".git", + ".hg", + ".tox", + ".nox", + ".pytest_cache", + ".mypy_cache", +} + + +def _iter_module_paths() -> t.Iterator[str]: + """Find the filesystem paths associated with imported modules.""" + # List is in case the value is modified by the app while updating. + for module in list(sys.modules.values()): + name = getattr(module, "__file__", None) + + if name is None or name.startswith(_ignore_always): + continue + + while not os.path.isfile(name): + # Zip file, find the base file without the module path. + old = name + name = os.path.dirname(name) + + if name == old: # skip if it was all directories somehow + break + else: + yield name + + +def _remove_by_pattern(paths: set[str], exclude_patterns: set[str]) -> None: + for pattern in exclude_patterns: + paths.difference_update(fnmatch.filter(paths, pattern)) + + +def _find_stat_paths( + extra_files: set[str], exclude_patterns: set[str] +) -> t.Iterable[str]: + """Find paths for the stat reloader to watch. Returns imported + module files, Python files under non-system paths. Extra files and + Python files under extra directories can also be scanned. + + System paths have to be excluded for efficiency. Non-system paths, + such as a project root or ``sys.path.insert``, should be the paths + of interest to the user anyway. + """ + paths = set() + + for path in chain(list(sys.path), extra_files): + path = os.path.abspath(path) + + if os.path.isfile(path): + # zip file on sys.path, or extra file + paths.add(path) + continue + + parent_has_py = {os.path.dirname(path): True} + + for root, dirs, files in os.walk(path): + if ( + root.startswith(_stat_ignore_scan) + or os.path.basename(root) in _ignore_common_dirs + ): + dirs.clear() + continue + + has_py = False + + for name in files: + if name.endswith((".py", ".pyc")): + has_py = True + paths.add(os.path.join(root, name)) + + # Optimization: stop scanning a directory if neither it nor + # its parent contained Python files. + if not (has_py or parent_has_py[os.path.dirname(root)]): + dirs.clear() + continue + + parent_has_py[root] = has_py + + paths.update(_iter_module_paths()) + _remove_by_pattern(paths, exclude_patterns) + return paths + + +def _find_watchdog_paths( + extra_files: set[str], exclude_patterns: set[str] +) -> t.Iterable[str]: + """Find paths for the stat reloader to watch. Looks at the same + sources as the stat reloader, but watches everything under + directories instead of individual files. + """ + dirs = set() + + for name in chain(list(sys.path), extra_files): + name = os.path.abspath(name) + + if os.path.isfile(name): + name = os.path.dirname(name) + + dirs.add(name) + + for name in _iter_module_paths(): + dirs.add(os.path.dirname(name)) + + _remove_by_pattern(dirs, exclude_patterns) + return _find_common_roots(dirs) + + +def _find_common_roots(paths: t.Iterable[str]) -> t.Iterable[str]: + root: dict[str, dict[str, t.Any]] = {} + + for chunks in sorted((PurePath(x).parts for x in paths), key=len, reverse=True): + node = root + + for chunk in chunks: + node = node.setdefault(chunk, {}) + + node.clear() + + rv = set() + + def _walk(node: t.Mapping[str, dict[str, t.Any]], path: tuple[str, ...]) -> None: + for prefix, child in node.items(): + _walk(child, path + (prefix,)) + + # If there are no more nodes, and a path has been accumulated, add it. + # Path may be empty if the "" entry is in sys.path. + if not node and path: + rv.add(os.path.join(*path)) + + _walk(root, ()) + return rv + + +def _get_args_for_reloading() -> list[str]: + """Determine how the script was executed, and return the args needed + to execute it again in a new process. + """ + if sys.version_info >= (3, 10): + # sys.orig_argv, added in Python 3.10, contains the exact args used to invoke + # Python. Still replace argv[0] with sys.executable for accuracy. + return [sys.executable, *sys.orig_argv[1:]] + + rv = [sys.executable] + py_script = sys.argv[0] + args = sys.argv[1:] + # Need to look at main module to determine how it was executed. + __main__ = sys.modules["__main__"] + + # The value of __package__ indicates how Python was called. It may + # not exist if a setuptools script is installed as an egg. It may be + # set incorrectly for entry points created with pip on Windows. + if getattr(__main__, "__package__", None) is None or ( + os.name == "nt" + and __main__.__package__ == "" + and not os.path.exists(py_script) + and os.path.exists(f"{py_script}.exe") + ): + # Executed a file, like "python app.py". + py_script = os.path.abspath(py_script) + + if os.name == "nt": + # Windows entry points have ".exe" extension and should be + # called directly. + if not os.path.exists(py_script) and os.path.exists(f"{py_script}.exe"): + py_script += ".exe" + + if ( + os.path.splitext(sys.executable)[1] == ".exe" + and os.path.splitext(py_script)[1] == ".exe" + ): + rv.pop(0) + + rv.append(py_script) + else: + # Executed a module, like "python -m werkzeug.serving". + if os.path.isfile(py_script): + # Rewritten by Python from "-m script" to "/path/to/script.py". + py_module = t.cast(str, __main__.__package__) + name = os.path.splitext(os.path.basename(py_script))[0] + + if name != "__main__": + py_module += f".{name}" + else: + # Incorrectly rewritten by pydevd debugger from "-m script" to "script". + py_module = py_script + + rv.extend(("-m", py_module.lstrip("."))) + + rv.extend(args) + return rv + + +class ReloaderLoop: + name = "" + + def __init__( + self, + extra_files: t.Iterable[str] | None = None, + exclude_patterns: t.Iterable[str] | None = None, + interval: int | float = 1, + ) -> None: + self.extra_files: set[str] = {os.path.abspath(x) for x in extra_files or ()} + self.exclude_patterns: set[str] = set(exclude_patterns or ()) + self.interval = interval + + def __enter__(self) -> ReloaderLoop: + """Do any setup, then run one step of the watch to populate the + initial filesystem state. + """ + self.run_step() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): # type: ignore + """Clean up any resources associated with the reloader.""" + pass + + def run(self) -> None: + """Continually run the watch step, sleeping for the configured + interval after each step. + """ + while True: + self.run_step() + time.sleep(self.interval) + + def run_step(self) -> None: + """Run one step for watching the filesystem. Called once to set + up initial state, then repeatedly to update it. + """ + pass + + def restart_with_reloader(self) -> int: + """Spawn a new Python interpreter with the same arguments as the + current one, but running the reloader thread. + """ + while True: + _log("info", f" * Restarting with {self.name}") + args = _get_args_for_reloading() + new_environ = os.environ.copy() + new_environ["WERKZEUG_RUN_MAIN"] = "true" + exit_code = subprocess.call(args, env=new_environ, close_fds=False) + + if exit_code != 3: + return exit_code + + def trigger_reload(self, filename: str) -> None: + self.log_reload(filename) + sys.exit(3) + + def log_reload(self, filename: str | bytes) -> None: + filename = os.path.abspath(filename) + _log("info", f" * Detected change in {filename!r}, reloading") + + +class StatReloaderLoop(ReloaderLoop): + name = "stat" + + def __enter__(self) -> ReloaderLoop: + self.mtimes: dict[str, float] = {} + return super().__enter__() + + def run_step(self) -> None: + for name in _find_stat_paths(self.extra_files, self.exclude_patterns): + try: + mtime = os.stat(name).st_mtime + except OSError: + continue + + old_time = self.mtimes.get(name) + + if old_time is None: + self.mtimes[name] = mtime + continue + + if mtime > old_time: + self.trigger_reload(name) + + +class WatchdogReloaderLoop(ReloaderLoop): + def __init__(self, *args: t.Any, **kwargs: t.Any) -> None: + from watchdog.events import EVENT_TYPE_CLOSED + from watchdog.events import EVENT_TYPE_CREATED + from watchdog.events import EVENT_TYPE_DELETED + from watchdog.events import EVENT_TYPE_MODIFIED + from watchdog.events import EVENT_TYPE_MOVED + from watchdog.events import FileModifiedEvent + from watchdog.events import PatternMatchingEventHandler + from watchdog.observers import Observer + + super().__init__(*args, **kwargs) + trigger_reload = self.trigger_reload + + class EventHandler(PatternMatchingEventHandler): + def on_any_event(self, event: FileModifiedEvent) -> None: # type: ignore[override] + if event.event_type not in { + EVENT_TYPE_CLOSED, + EVENT_TYPE_CREATED, + EVENT_TYPE_DELETED, + EVENT_TYPE_MODIFIED, + EVENT_TYPE_MOVED, + }: + # skip events that don't involve changes to the file + return + + trigger_reload(event.src_path) + + reloader_name = Observer.__name__.lower() # type: ignore[attr-defined] + + if reloader_name.endswith("observer"): + reloader_name = reloader_name[:-8] + + self.name = f"watchdog ({reloader_name})" + self.observer = Observer() + extra_patterns = (p for p in self.extra_files if not os.path.isdir(p)) + self.event_handler = EventHandler( + patterns=["*.py", "*.pyc", "*.zip", *extra_patterns], + ignore_patterns=[ + *[f"*/{d}/*" for d in _ignore_common_dirs], + *self.exclude_patterns, + ], + ) + self.should_reload = threading.Event() + + def trigger_reload(self, filename: str | bytes) -> None: + # This is called inside an event handler, which means throwing + # SystemExit has no effect. + # https://github.com/gorakhargosh/watchdog/issues/294 + self.should_reload.set() + self.log_reload(filename) + + def __enter__(self) -> ReloaderLoop: + self.watches: dict[str, t.Any] = {} + self.observer.start() + return super().__enter__() + + def __exit__(self, exc_type, exc_val, exc_tb): # type: ignore + self.observer.stop() + self.observer.join() + + def run(self) -> None: + while not self.should_reload.wait(timeout=self.interval): + self.run_step() + + sys.exit(3) + + def run_step(self) -> None: + to_delete = set(self.watches) + + for path in _find_watchdog_paths(self.extra_files, self.exclude_patterns): + if path not in self.watches: + try: + self.watches[path] = self.observer.schedule( + self.event_handler, path, recursive=True + ) + except OSError: + # Clear this path from list of watches. We don't want + # the same error message showing again in the next + # iteration. + self.watches[path] = None + + to_delete.discard(path) + + for path in to_delete: + watch = self.watches.pop(path, None) + + if watch is not None: + self.observer.unschedule(watch) + + +reloader_loops: dict[str, type[ReloaderLoop]] = { + "stat": StatReloaderLoop, + "watchdog": WatchdogReloaderLoop, +} + +try: + __import__("watchdog.observers") +except ImportError: + reloader_loops["auto"] = reloader_loops["stat"] +else: + reloader_loops["auto"] = reloader_loops["watchdog"] + + +def ensure_echo_on() -> None: + """Ensure that echo mode is enabled. Some tools such as PDB disable + it which causes usability issues after a reload.""" + # tcgetattr will fail if stdin isn't a tty + if sys.stdin is None or not sys.stdin.isatty(): + return + + try: + import termios + except ImportError: + return + + attributes = termios.tcgetattr(sys.stdin) + + if not attributes[3] & termios.ECHO: + attributes[3] |= termios.ECHO + termios.tcsetattr(sys.stdin, termios.TCSANOW, attributes) + + +def run_with_reloader( + main_func: t.Callable[[], None], + extra_files: t.Iterable[str] | None = None, + exclude_patterns: t.Iterable[str] | None = None, + interval: int | float = 1, + reloader_type: str = "auto", +) -> None: + """Run the given function in an independent Python interpreter.""" + import signal + + signal.signal(signal.SIGTERM, lambda *args: sys.exit(0)) + reloader = reloader_loops[reloader_type]( + extra_files=extra_files, exclude_patterns=exclude_patterns, interval=interval + ) + + try: + if os.environ.get("WERKZEUG_RUN_MAIN") == "true": + ensure_echo_on() + t = threading.Thread(target=main_func, args=()) + t.daemon = True + + # Enter the reloader to set up initial state, then start + # the app thread and reloader update loop. + with reloader: + t.start() + reloader.run() + else: + sys.exit(reloader.restart_with_reloader()) + except KeyboardInterrupt: + pass diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__init__.py b/lib/python3.12/site-packages/werkzeug/datastructures/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6582de02c78b54a785bfbc0ec226e33d2ef48785 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/__init__.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import typing as t + +from .accept import Accept as Accept +from .accept import CharsetAccept as CharsetAccept +from .accept import LanguageAccept as LanguageAccept +from .accept import MIMEAccept as MIMEAccept +from .auth import Authorization as Authorization +from .auth import WWWAuthenticate as WWWAuthenticate +from .cache_control import RequestCacheControl as RequestCacheControl +from .cache_control import ResponseCacheControl as ResponseCacheControl +from .csp import ContentSecurityPolicy as ContentSecurityPolicy +from .etag import ETags as ETags +from .file_storage import FileMultiDict as FileMultiDict +from .file_storage import FileStorage as FileStorage +from .headers import EnvironHeaders as EnvironHeaders +from .headers import Headers as Headers +from .mixins import ImmutableDictMixin as ImmutableDictMixin +from .mixins import ImmutableHeadersMixin as ImmutableHeadersMixin +from .mixins import ImmutableListMixin as ImmutableListMixin +from .mixins import ImmutableMultiDictMixin as ImmutableMultiDictMixin +from .mixins import UpdateDictMixin as UpdateDictMixin +from .range import ContentRange as ContentRange +from .range import IfRange as IfRange +from .range import Range as Range +from .structures import CallbackDict as CallbackDict +from .structures import CombinedMultiDict as CombinedMultiDict +from .structures import HeaderSet as HeaderSet +from .structures import ImmutableDict as ImmutableDict +from .structures import ImmutableList as ImmutableList +from .structures import ImmutableMultiDict as ImmutableMultiDict +from .structures import ImmutableTypeConversionDict as ImmutableTypeConversionDict +from .structures import iter_multi_items as iter_multi_items +from .structures import MultiDict as MultiDict +from .structures import TypeConversionDict as TypeConversionDict + + +def __getattr__(name: str) -> t.Any: + import warnings + + if name == "OrderedMultiDict": + from .structures import _OrderedMultiDict + + warnings.warn( + "'OrderedMultiDict' is deprecated and will be removed in Werkzeug" + " 3.2. Use 'MultiDict' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _OrderedMultiDict + + if name == "ImmutableOrderedMultiDict": + from .structures import _ImmutableOrderedMultiDict + + warnings.warn( + "'OrderedMultiDict' is deprecated and will be removed in Werkzeug" + " 3.2. Use 'ImmutableMultiDict' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _ImmutableOrderedMultiDict + + raise AttributeError(name) diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17bfbfc351427d39b5a4951044c6f07bcf37beb4 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/accept.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/accept.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08fca3fa179c960cb31e121f0301db851d95be65 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/accept.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/auth.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/auth.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..188d4f47f3d99bc9ee284fdeac9f2311f1e5e5e3 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/auth.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/cache_control.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/cache_control.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..095478a01dcfb5206a625c4d1c0414c6bd68f40f Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/cache_control.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/csp.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/csp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3674ad7082c6d6b1343249b16bf10d4384b701d Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/csp.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/etag.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/etag.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4e12cfbe870e96845cb43db52f9e7e2e75c8e755 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/etag.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/file_storage.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/file_storage.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..122ae5d0eea8b1678071ebd43b507f5edf6eebcf Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/file_storage.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/headers.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/headers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9c33464e61575a55a7ba3b939a3da0f2f874d898 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/headers.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/mixins.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/mixins.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..545263370f21c86abf3014e7c89e492908815f99 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/mixins.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/range.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/range.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9507133bc432da1d6f221d9acef0ed8df55f9bac Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/range.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/structures.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/structures.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..75030c380c2f5562aa5655c1c4c993d0fe4024af Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/datastructures/__pycache__/structures.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/accept.py b/lib/python3.12/site-packages/werkzeug/datastructures/accept.py new file mode 100644 index 0000000000000000000000000000000000000000..44179a93f5f3d96bd50bc2c0da991ce048b8f530 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/accept.py @@ -0,0 +1,350 @@ +from __future__ import annotations + +import codecs +import collections.abc as cabc +import re +import typing as t + +from .structures import ImmutableList + + +class Accept(ImmutableList[tuple[str, float]]): + """An :class:`Accept` object is just a list subclass for lists of + ``(value, quality)`` tuples. It is automatically sorted by specificity + and quality. + + All :class:`Accept` objects work similar to a list but provide extra + functionality for working with the data. Containment checks are + normalized to the rules of that header: + + >>> a = CharsetAccept([('ISO-8859-1', 1), ('utf-8', 0.7)]) + >>> a.best + 'ISO-8859-1' + >>> 'iso-8859-1' in a + True + >>> 'UTF8' in a + True + >>> 'utf7' in a + False + + To get the quality for an item you can use normal item lookup: + + >>> print a['utf-8'] + 0.7 + >>> a['utf7'] + 0 + + .. versionchanged:: 0.5 + :class:`Accept` objects are forced immutable now. + + .. versionchanged:: 1.0.0 + :class:`Accept` internal values are no longer ordered + alphabetically for equal quality tags. Instead the initial + order is preserved. + + """ + + def __init__( + self, values: Accept | cabc.Iterable[tuple[str, float]] | None = () + ) -> None: + if values is None: + super().__init__() + self.provided = False + elif isinstance(values, Accept): + self.provided = values.provided + super().__init__(values) + else: + self.provided = True + values = sorted( + values, key=lambda x: (self._specificity(x[0]), x[1]), reverse=True + ) + super().__init__(values) + + def _specificity(self, value: str) -> tuple[bool, ...]: + """Returns a tuple describing the value's specificity.""" + return (value != "*",) + + def _value_matches(self, value: str, item: str) -> bool: + """Check if a value matches a given accept item.""" + return item == "*" or item.lower() == value.lower() + + @t.overload + def __getitem__(self, key: str) -> float: ... + @t.overload + def __getitem__(self, key: t.SupportsIndex) -> tuple[str, float]: ... + @t.overload + def __getitem__(self, key: slice) -> list[tuple[str, float]]: ... + def __getitem__( + self, key: str | t.SupportsIndex | slice + ) -> float | tuple[str, float] | list[tuple[str, float]]: + """Besides index lookup (getting item n) you can also pass it a string + to get the quality for the item. If the item is not in the list, the + returned quality is ``0``. + """ + if isinstance(key, str): + return self.quality(key) + return list.__getitem__(self, key) + + def quality(self, key: str) -> float: + """Returns the quality of the key. + + .. versionadded:: 0.6 + In previous versions you had to use the item-lookup syntax + (eg: ``obj[key]`` instead of ``obj.quality(key)``) + """ + for item, quality in self: + if self._value_matches(key, item): + return quality + return 0 + + def __contains__(self, value: str) -> bool: # type: ignore[override] + for item, _quality in self: + if self._value_matches(value, item): + return True + return False + + def __repr__(self) -> str: + pairs_str = ", ".join(f"({x!r}, {y})" for x, y in self) + return f"{type(self).__name__}([{pairs_str}])" + + def index(self, key: str | tuple[str, float]) -> int: # type: ignore[override] + """Get the position of an entry or raise :exc:`ValueError`. + + :param key: The key to be looked up. + + .. versionchanged:: 0.5 + This used to raise :exc:`IndexError`, which was inconsistent + with the list API. + """ + if isinstance(key, str): + for idx, (item, _quality) in enumerate(self): + if self._value_matches(key, item): + return idx + raise ValueError(key) + return list.index(self, key) + + def find(self, key: str | tuple[str, float]) -> int: + """Get the position of an entry or return -1. + + :param key: The key to be looked up. + """ + try: + return self.index(key) + except ValueError: + return -1 + + def values(self) -> cabc.Iterator[str]: + """Iterate over all values.""" + for item in self: + yield item[0] + + def to_header(self) -> str: + """Convert the header set into an HTTP header string.""" + result = [] + for value, quality in self: + if quality != 1: + value = f"{value};q={quality}" + result.append(value) + return ",".join(result) + + def __str__(self) -> str: + return self.to_header() + + def _best_single_match(self, match: str) -> tuple[str, float] | None: + for client_item, quality in self: + if self._value_matches(match, client_item): + # self is sorted by specificity descending, we can exit + return client_item, quality + return None + + @t.overload + def best_match(self, matches: cabc.Iterable[str]) -> str | None: ... + @t.overload + def best_match(self, matches: cabc.Iterable[str], default: str = ...) -> str: ... + def best_match( + self, matches: cabc.Iterable[str], default: str | None = None + ) -> str | None: + """Returns the best match from a list of possible matches based + on the specificity and quality of the client. If two items have the + same quality and specificity, the one is returned that comes first. + + :param matches: a list of matches to check for + :param default: the value that is returned if none match + """ + result = default + best_quality: float = -1 + best_specificity: tuple[float, ...] = (-1,) + for server_item in matches: + match = self._best_single_match(server_item) + if not match: + continue + client_item, quality = match + specificity = self._specificity(client_item) + if quality <= 0 or quality < best_quality: + continue + # better quality or same quality but more specific => better match + if quality > best_quality or specificity > best_specificity: + result = server_item + best_quality = quality + best_specificity = specificity + return result + + @property + def best(self) -> str | None: + """The best match as value.""" + if self: + return self[0][0] + + return None + + +_mime_split_re = re.compile(r"/|(?:\s*;\s*)") + + +def _normalize_mime(value: str) -> list[str]: + return _mime_split_re.split(value.lower()) + + +class MIMEAccept(Accept): + """Like :class:`Accept` but with special methods and behavior for + mimetypes. + """ + + def _specificity(self, value: str) -> tuple[bool, ...]: + return tuple(x != "*" for x in _mime_split_re.split(value)) + + def _value_matches(self, value: str, item: str) -> bool: + # item comes from the client, can't match if it's invalid. + if "/" not in item: + return False + + # value comes from the application, tell the developer when it + # doesn't look valid. + if "/" not in value: + raise ValueError(f"invalid mimetype {value!r}") + + # Split the match value into type, subtype, and a sorted list of parameters. + normalized_value = _normalize_mime(value) + value_type, value_subtype = normalized_value[:2] + value_params = sorted(normalized_value[2:]) + + # "*/*" is the only valid value that can start with "*". + if value_type == "*" and value_subtype != "*": + raise ValueError(f"invalid mimetype {value!r}") + + # Split the accept item into type, subtype, and parameters. + normalized_item = _normalize_mime(item) + item_type, item_subtype = normalized_item[:2] + item_params = sorted(normalized_item[2:]) + + # "*/not-*" from the client is invalid, can't match. + if item_type == "*" and item_subtype != "*": + return False + + return ( + (item_type == "*" and item_subtype == "*") + or (value_type == "*" and value_subtype == "*") + ) or ( + item_type == value_type + and ( + item_subtype == "*" + or value_subtype == "*" + or (item_subtype == value_subtype and item_params == value_params) + ) + ) + + @property + def accept_html(self) -> bool: + """True if this object accepts HTML.""" + return "text/html" in self or self.accept_xhtml # type: ignore[comparison-overlap] + + @property + def accept_xhtml(self) -> bool: + """True if this object accepts XHTML.""" + return "application/xhtml+xml" in self or "application/xml" in self # type: ignore[comparison-overlap] + + @property + def accept_json(self) -> bool: + """True if this object accepts JSON.""" + return "application/json" in self # type: ignore[comparison-overlap] + + +_locale_delim_re = re.compile(r"[_-]") + + +def _normalize_lang(value: str) -> list[str]: + """Process a language tag for matching.""" + return _locale_delim_re.split(value.lower()) + + +class LanguageAccept(Accept): + """Like :class:`Accept` but with normalization for language tags.""" + + def _value_matches(self, value: str, item: str) -> bool: + return item == "*" or _normalize_lang(value) == _normalize_lang(item) + + @t.overload + def best_match(self, matches: cabc.Iterable[str]) -> str | None: ... + @t.overload + def best_match(self, matches: cabc.Iterable[str], default: str = ...) -> str: ... + def best_match( + self, matches: cabc.Iterable[str], default: str | None = None + ) -> str | None: + """Given a list of supported values, finds the best match from + the list of accepted values. + + Language tags are normalized for the purpose of matching, but + are returned unchanged. + + If no exact match is found, this will fall back to matching + the first subtag (primary language only), first with the + accepted values then with the match values. This partial is not + applied to any other language subtags. + + The default is returned if no exact or fallback match is found. + + :param matches: A list of supported languages to find a match. + :param default: The value that is returned if none match. + """ + # Look for an exact match first. If a client accepts "en-US", + # "en-US" is a valid match at this point. + result = super().best_match(matches) + + if result is not None: + return result + + # Fall back to accepting primary tags. If a client accepts + # "en-US", "en" is a valid match at this point. Need to use + # re.split to account for 2 or 3 letter codes. + fallback = Accept( + [(_locale_delim_re.split(item[0], 1)[0], item[1]) for item in self] + ) + result = fallback.best_match(matches) + + if result is not None: + return result + + # Fall back to matching primary tags. If the client accepts + # "en", "en-US" is a valid match at this point. + fallback_matches = [_locale_delim_re.split(item, 1)[0] for item in matches] + result = super().best_match(fallback_matches) + + # Return a value from the original match list. Find the first + # original value that starts with the matched primary tag. + if result is not None: + return next(item for item in matches if item.startswith(result)) + + return default + + +class CharsetAccept(Accept): + """Like :class:`Accept` but with normalization for charsets.""" + + def _value_matches(self, value: str, item: str) -> bool: + def _normalize(name: str) -> str: + try: + return codecs.lookup(name).name + except LookupError: + return name.lower() + + return item == "*" or _normalize(value) == _normalize(item) diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/auth.py b/lib/python3.12/site-packages/werkzeug/datastructures/auth.py new file mode 100644 index 0000000000000000000000000000000000000000..42f7aa4683bcbd612fc8c5032572d5c48d3fc99e --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/auth.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import base64 +import binascii +import collections.abc as cabc +import typing as t + +from ..http import dump_header +from ..http import parse_dict_header +from ..http import quote_header_value +from .structures import CallbackDict + +if t.TYPE_CHECKING: + import typing_extensions as te + + +class Authorization: + """Represents the parts of an ``Authorization`` request header. + + :attr:`.Request.authorization` returns an instance if the header is set. + + An instance can be used with the test :class:`.Client` request methods' ``auth`` + parameter to send the header in test requests. + + Depending on the auth scheme, either :attr:`parameters` or :attr:`token` will be + set. The ``Basic`` scheme's token is decoded into the ``username`` and ``password`` + parameters. + + For convenience, ``auth["key"]`` and ``auth.key`` both access the key in the + :attr:`parameters` dict, along with ``auth.get("key")`` and ``"key" in auth``. + + .. versionchanged:: 2.3 + The ``token`` parameter and attribute was added to support auth schemes that use + a token instead of parameters, such as ``Bearer``. + + .. versionchanged:: 2.3 + The object is no longer a ``dict``. + + .. versionchanged:: 0.5 + The object is an immutable dict. + """ + + def __init__( + self, + auth_type: str, + data: dict[str, str | None] | None = None, + token: str | None = None, + ) -> None: + self.type = auth_type + """The authorization scheme, like ``basic``, ``digest``, or ``bearer``.""" + + if data is None: + data = {} + + self.parameters = data + """A dict of parameters parsed from the header. Either this or :attr:`token` + will have a value for a given scheme. + """ + + self.token = token + """A token parsed from the header. Either this or :attr:`parameters` will have a + value for a given scheme. + + .. versionadded:: 2.3 + """ + + def __getattr__(self, name: str) -> str | None: + return self.parameters.get(name) + + def __getitem__(self, name: str) -> str | None: + return self.parameters.get(name) + + def get(self, key: str, default: str | None = None) -> str | None: + return self.parameters.get(key, default) + + def __contains__(self, key: str) -> bool: + return key in self.parameters + + def __eq__(self, other: object) -> bool: + if not isinstance(other, Authorization): + return NotImplemented + + return ( + other.type == self.type + and other.token == self.token + and other.parameters == self.parameters + ) + + @classmethod + def from_header(cls, value: str | None) -> te.Self | None: + """Parse an ``Authorization`` header value and return an instance, or ``None`` + if the value is empty. + + :param value: The header value to parse. + + .. versionadded:: 2.3 + """ + if not value: + return None + + scheme, _, rest = value.partition(" ") + scheme = scheme.lower() + rest = rest.strip() + + if scheme == "basic": + try: + username, _, password = base64.b64decode(rest).decode().partition(":") + except (binascii.Error, UnicodeError): + return None + + return cls(scheme, {"username": username, "password": password}) + + if "=" in rest.rstrip("="): + # = that is not trailing, this is parameters. + return cls(scheme, parse_dict_header(rest), None) + + # No = or only trailing =, this is a token. + return cls(scheme, None, rest) + + def to_header(self) -> str: + """Produce an ``Authorization`` header value representing this data. + + .. versionadded:: 2.0 + """ + if self.type == "basic": + value = base64.b64encode( + f"{self.username}:{self.password}".encode() + ).decode("ascii") + return f"Basic {value}" + + if self.token is not None: + return f"{self.type.title()} {self.token}" + + return f"{self.type.title()} {dump_header(self.parameters)}" + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + return f"<{type(self).__name__} {self.to_header()}>" + + +class WWWAuthenticate: + """Represents the parts of a ``WWW-Authenticate`` response header. + + Set :attr:`.Response.www_authenticate` to an instance of list of instances to set + values for this header in the response. Modifying this instance will modify the + header value. + + Depending on the auth scheme, either :attr:`parameters` or :attr:`token` should be + set. The ``Basic`` scheme will encode ``username`` and ``password`` parameters to a + token. + + For convenience, ``auth["key"]`` and ``auth.key`` both act on the :attr:`parameters` + dict, and can be used to get, set, or delete parameters. ``auth.get("key")`` and + ``"key" in auth`` are also provided. + + .. versionchanged:: 2.3 + The ``token`` parameter and attribute was added to support auth schemes that use + a token instead of parameters, such as ``Bearer``. + + .. versionchanged:: 2.3 + The object is no longer a ``dict``. + + .. versionchanged:: 2.3 + The ``on_update`` parameter was removed. + """ + + def __init__( + self, + auth_type: str, + values: dict[str, str | None] | None = None, + token: str | None = None, + ): + self._type = auth_type.lower() + self._parameters: dict[str, str | None] = CallbackDict( + values, lambda _: self._trigger_on_update() + ) + self._token = token + self._on_update: cabc.Callable[[WWWAuthenticate], None] | None = None + + def _trigger_on_update(self) -> None: + if self._on_update is not None: + self._on_update(self) + + @property + def type(self) -> str: + """The authorization scheme, like ``basic``, ``digest``, or ``bearer``.""" + return self._type + + @type.setter + def type(self, value: str) -> None: + self._type = value + self._trigger_on_update() + + @property + def parameters(self) -> dict[str, str | None]: + """A dict of parameters for the header. Only one of this or :attr:`token` should + have a value for a given scheme. + """ + return self._parameters + + @parameters.setter + def parameters(self, value: dict[str, str]) -> None: + self._parameters = CallbackDict(value, lambda _: self._trigger_on_update()) + self._trigger_on_update() + + @property + def token(self) -> str | None: + """A dict of parameters for the header. Only one of this or :attr:`token` should + have a value for a given scheme. + """ + return self._token + + @token.setter + def token(self, value: str | None) -> None: + """A token for the header. Only one of this or :attr:`parameters` should have a + value for a given scheme. + + .. versionadded:: 2.3 + """ + self._token = value + self._trigger_on_update() + + def __getitem__(self, key: str) -> str | None: + return self.parameters.get(key) + + def __setitem__(self, key: str, value: str | None) -> None: + if value is None: + if key in self.parameters: + del self.parameters[key] + else: + self.parameters[key] = value + + self._trigger_on_update() + + def __delitem__(self, key: str) -> None: + if key in self.parameters: + del self.parameters[key] + self._trigger_on_update() + + def __getattr__(self, name: str) -> str | None: + return self[name] + + def __setattr__(self, name: str, value: str | None) -> None: + if name in {"_type", "_parameters", "_token", "_on_update"}: + super().__setattr__(name, value) + else: + self[name] = value + + def __delattr__(self, name: str) -> None: + del self[name] + + def __contains__(self, key: str) -> bool: + return key in self.parameters + + def __eq__(self, other: object) -> bool: + if not isinstance(other, WWWAuthenticate): + return NotImplemented + + return ( + other.type == self.type + and other.token == self.token + and other.parameters == self.parameters + ) + + def get(self, key: str, default: str | None = None) -> str | None: + return self.parameters.get(key, default) + + @classmethod + def from_header(cls, value: str | None) -> te.Self | None: + """Parse a ``WWW-Authenticate`` header value and return an instance, or ``None`` + if the value is empty. + + :param value: The header value to parse. + + .. versionadded:: 2.3 + """ + if not value: + return None + + scheme, _, rest = value.partition(" ") + scheme = scheme.lower() + rest = rest.strip() + + if "=" in rest.rstrip("="): + # = that is not trailing, this is parameters. + return cls(scheme, parse_dict_header(rest), None) + + # No = or only trailing =, this is a token. + return cls(scheme, None, rest) + + def to_header(self) -> str: + """Produce a ``WWW-Authenticate`` header value representing this data.""" + if self.token is not None: + return f"{self.type.title()} {self.token}" + + if self.type == "digest": + items = [] + + for key, value in self.parameters.items(): + if key in {"realm", "domain", "nonce", "opaque", "qop"}: + value = quote_header_value(value, allow_token=False) + else: + value = quote_header_value(value) + + items.append(f"{key}={value}") + + return f"Digest {', '.join(items)}" + + return f"{self.type.title()} {dump_header(self.parameters)}" + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + return f"<{type(self).__name__} {self.to_header()}>" diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/cache_control.py b/lib/python3.12/site-packages/werkzeug/datastructures/cache_control.py new file mode 100644 index 0000000000000000000000000000000000000000..3d963102825b036ebcd9a441b01dcdd5df1390b6 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/cache_control.py @@ -0,0 +1,273 @@ +from __future__ import annotations + +import collections.abc as cabc +import typing as t +from inspect import cleandoc + +from .mixins import ImmutableDictMixin +from .structures import CallbackDict + + +def cache_control_property( + key: str, empty: t.Any, type: type[t.Any] | None, *, doc: str | None = None +) -> t.Any: + """Return a new property object for a cache header. Useful if you + want to add support for a cache extension in a subclass. + + :param key: The attribute name present in the parsed cache-control header dict. + :param empty: The value to use if the key is present without a value. + :param type: The type to convert the string value to instead of a string. If + conversion raises a ``ValueError``, the returned value is ``None``. + :param doc: The docstring for the property. If not given, it is generated + based on the other params. + + .. versionchanged:: 3.1 + Added the ``doc`` param. + + .. versionchanged:: 2.0 + Renamed from ``cache_property``. + """ + if doc is None: + parts = [f"The ``{key}`` attribute."] + + if type is bool: + parts.append("A ``bool``, either present or not.") + else: + if type is None: + parts.append("A ``str``,") + else: + parts.append(f"A ``{type.__name__}``,") + + if empty is not None: + parts.append(f"``{empty!r}`` if present with no value,") + + parts.append("or ``None`` if not present.") + + doc = " ".join(parts) + + return property( + lambda x: x._get_cache_value(key, empty, type), + lambda x, v: x._set_cache_value(key, v, type), + lambda x: x._del_cache_value(key), + doc=cleandoc(doc), + ) + + +class _CacheControl(CallbackDict[str, t.Optional[str]]): + """Subclass of a dict that stores values for a Cache-Control header. It + has accessors for all the cache-control directives specified in RFC 2616. + The class does not differentiate between request and response directives. + + Because the cache-control directives in the HTTP header use dashes the + python descriptors use underscores for that. + + To get a header of the :class:`CacheControl` object again you can convert + the object into a string or call the :meth:`to_header` method. If you plan + to subclass it and add your own items have a look at the sourcecode for + that class. + + .. versionchanged:: 3.1 + Dict values are always ``str | None``. Setting properties will + convert the value to a string. Setting a non-bool property to + ``False`` is equivalent to setting it to ``None``. Getting typed + properties will return ``None`` if conversion raises + ``ValueError``, rather than the string. + + .. versionchanged:: 2.1 + Setting int properties such as ``max_age`` will convert the + value to an int. + + .. versionchanged:: 0.4 + Setting ``no_cache`` or ``private`` to ``True`` will set the + implicit value ``"*"``. + """ + + no_store: bool = cache_control_property("no-store", None, bool) + max_age: int | None = cache_control_property("max-age", None, int) + no_transform: bool = cache_control_property("no-transform", None, bool) + stale_if_error: int | None = cache_control_property("stale-if-error", None, int) + + def __init__( + self, + values: cabc.Mapping[str, t.Any] | cabc.Iterable[tuple[str, t.Any]] | None = (), + on_update: cabc.Callable[[_CacheControl], None] | None = None, + ): + super().__init__(values, on_update) + self.provided = values is not None + + def _get_cache_value( + self, key: str, empty: t.Any, type: type[t.Any] | None + ) -> t.Any: + """Used internally by the accessor properties.""" + if type is bool: + return key in self + + if key not in self: + return None + + if (value := self[key]) is None: + return empty + + if type is not None: + try: + value = type(value) + except ValueError: + return None + + return value + + def _set_cache_value( + self, key: str, value: t.Any, type: type[t.Any] | None + ) -> None: + """Used internally by the accessor properties.""" + if type is bool: + if value: + self[key] = None + else: + self.pop(key, None) + elif value is None or value is False: + self.pop(key, None) + elif value is True: + self[key] = None + else: + if type is not None: + value = type(value) + + self[key] = str(value) + + def _del_cache_value(self, key: str) -> None: + """Used internally by the accessor properties.""" + if key in self: + del self[key] + + def to_header(self) -> str: + """Convert the stored values into a cache control header.""" + return http.dump_header(self) + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + kv_str = " ".join(f"{k}={v!r}" for k, v in sorted(self.items())) + return f"<{type(self).__name__} {kv_str}>" + + cache_property = staticmethod(cache_control_property) + + +class RequestCacheControl(ImmutableDictMixin[str, t.Optional[str]], _CacheControl): # type: ignore[misc] + """A cache control for requests. This is immutable and gives access + to all the request-relevant cache control headers. + + To get a header of the :class:`RequestCacheControl` object again you can + convert the object into a string or call the :meth:`to_header` method. If + you plan to subclass it and add your own items have a look at the sourcecode + for that class. + + .. versionchanged:: 3.1 + Dict values are always ``str | None``. Setting properties will + convert the value to a string. Setting a non-bool property to + ``False`` is equivalent to setting it to ``None``. Getting typed + properties will return ``None`` if conversion raises + ``ValueError``, rather than the string. + + .. versionchanged:: 3.1 + ``max_age`` is ``None`` if present without a value, rather + than ``-1``. + + .. versionchanged:: 3.1 + ``no_cache`` is a boolean, it is ``True`` instead of ``"*"`` + when present. + + .. versionchanged:: 3.1 + ``max_stale`` is ``True`` if present without a value, rather + than ``"*"``. + + .. versionchanged:: 3.1 + ``no_transform`` is a boolean. Previously it was mistakenly + always ``None``. + + .. versionchanged:: 3.1 + ``min_fresh`` is ``None`` if present without a value, rather + than ``"*"``. + + .. versionchanged:: 2.1 + Setting int properties such as ``max_age`` will convert the + value to an int. + + .. versionadded:: 0.5 + Response-only properties are not present on this request class. + """ + + no_cache: bool = cache_control_property("no-cache", None, bool) + max_stale: int | t.Literal[True] | None = cache_control_property( + "max-stale", + True, + int, + ) + min_fresh: int | None = cache_control_property("min-fresh", None, int) + only_if_cached: bool = cache_control_property("only-if-cached", None, bool) + + +class ResponseCacheControl(_CacheControl): + """A cache control for responses. Unlike :class:`RequestCacheControl` + this is mutable and gives access to response-relevant cache control + headers. + + To get a header of the :class:`ResponseCacheControl` object again you can + convert the object into a string or call the :meth:`to_header` method. If + you plan to subclass it and add your own items have a look at the sourcecode + for that class. + + .. versionchanged:: 3.1 + Dict values are always ``str | None``. Setting properties will + convert the value to a string. Setting a non-bool property to + ``False`` is equivalent to setting it to ``None``. Getting typed + properties will return ``None`` if conversion raises + ``ValueError``, rather than the string. + + .. versionchanged:: 3.1 + ``no_cache`` is ``True`` if present without a value, rather than + ``"*"``. + + .. versionchanged:: 3.1 + ``private`` is ``True`` if present without a value, rather than + ``"*"``. + + .. versionchanged:: 3.1 + ``no_transform`` is a boolean. Previously it was mistakenly + always ``None``. + + .. versionchanged:: 3.1 + Added the ``must_understand``, ``stale_while_revalidate``, and + ``stale_if_error`` properties. + + .. versionchanged:: 2.1.1 + ``s_maxage`` converts the value to an int. + + .. versionchanged:: 2.1 + Setting int properties such as ``max_age`` will convert the + value to an int. + + .. versionadded:: 0.5 + Request-only properties are not present on this response class. + """ + + no_cache: str | t.Literal[True] | None = cache_control_property( + "no-cache", True, None + ) + public: bool = cache_control_property("public", None, bool) + private: str | t.Literal[True] | None = cache_control_property( + "private", True, None + ) + must_revalidate: bool = cache_control_property("must-revalidate", None, bool) + proxy_revalidate: bool = cache_control_property("proxy-revalidate", None, bool) + s_maxage: int | None = cache_control_property("s-maxage", None, int) + immutable: bool = cache_control_property("immutable", None, bool) + must_understand: bool = cache_control_property("must-understand", None, bool) + stale_while_revalidate: int | None = cache_control_property( + "stale-while-revalidate", None, int + ) + + +# circular dependencies +from .. import http # noqa: E402 diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/csp.py b/lib/python3.12/site-packages/werkzeug/datastructures/csp.py new file mode 100644 index 0000000000000000000000000000000000000000..206c1e38b2db03e1232906bf0a689f459a2ee3ca --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/csp.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import collections.abc as cabc +import typing as t + +from .structures import CallbackDict + + +def csp_property(key: str) -> t.Any: + """Return a new property object for a content security policy header. + Useful if you want to add support for a csp extension in a + subclass. + """ + return property( + lambda x: x._get_value(key), + lambda x, v: x._set_value(key, v), + lambda x: x._del_value(key), + f"accessor for {key!r}", + ) + + +class ContentSecurityPolicy(CallbackDict[str, str]): + """Subclass of a dict that stores values for a Content Security Policy + header. It has accessors for all the level 3 policies. + + Because the csp directives in the HTTP header use dashes the + python descriptors use underscores for that. + + To get a header of the :class:`ContentSecurityPolicy` object again + you can convert the object into a string or call the + :meth:`to_header` method. If you plan to subclass it and add your + own items have a look at the sourcecode for that class. + + .. versionadded:: 1.0.0 + Support for Content Security Policy headers was added. + + """ + + base_uri: str | None = csp_property("base-uri") + child_src: str | None = csp_property("child-src") + connect_src: str | None = csp_property("connect-src") + default_src: str | None = csp_property("default-src") + font_src: str | None = csp_property("font-src") + form_action: str | None = csp_property("form-action") + frame_ancestors: str | None = csp_property("frame-ancestors") + frame_src: str | None = csp_property("frame-src") + img_src: str | None = csp_property("img-src") + manifest_src: str | None = csp_property("manifest-src") + media_src: str | None = csp_property("media-src") + navigate_to: str | None = csp_property("navigate-to") + object_src: str | None = csp_property("object-src") + prefetch_src: str | None = csp_property("prefetch-src") + plugin_types: str | None = csp_property("plugin-types") + report_to: str | None = csp_property("report-to") + report_uri: str | None = csp_property("report-uri") + sandbox: str | None = csp_property("sandbox") + script_src: str | None = csp_property("script-src") + script_src_attr: str | None = csp_property("script-src-attr") + script_src_elem: str | None = csp_property("script-src-elem") + style_src: str | None = csp_property("style-src") + style_src_attr: str | None = csp_property("style-src-attr") + style_src_elem: str | None = csp_property("style-src-elem") + worker_src: str | None = csp_property("worker-src") + + def __init__( + self, + values: cabc.Mapping[str, str] | cabc.Iterable[tuple[str, str]] | None = (), + on_update: cabc.Callable[[ContentSecurityPolicy], None] | None = None, + ) -> None: + super().__init__(values, on_update) + self.provided = values is not None + + def _get_value(self, key: str) -> str | None: + """Used internally by the accessor properties.""" + return self.get(key) + + def _set_value(self, key: str, value: str | None) -> None: + """Used internally by the accessor properties.""" + if value is None: + self.pop(key, None) + else: + self[key] = value + + def _del_value(self, key: str) -> None: + """Used internally by the accessor properties.""" + if key in self: + del self[key] + + def to_header(self) -> str: + """Convert the stored values into a cache control header.""" + from ..http import dump_csp_header + + return dump_csp_header(self) + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + kv_str = " ".join(f"{k}={v!r}" for k, v in sorted(self.items())) + return f"<{type(self).__name__} {kv_str}>" diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/etag.py b/lib/python3.12/site-packages/werkzeug/datastructures/etag.py new file mode 100644 index 0000000000000000000000000000000000000000..a4ef342456480e3c08c870400be5cd0515ffd666 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/etag.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import collections.abc as cabc + + +class ETags(cabc.Collection[str]): + """A set that can be used to check if one etag is present in a collection + of etags. + """ + + def __init__( + self, + strong_etags: cabc.Iterable[str] | None = None, + weak_etags: cabc.Iterable[str] | None = None, + star_tag: bool = False, + ): + if not star_tag and strong_etags: + self._strong = frozenset(strong_etags) + else: + self._strong = frozenset() + + self._weak = frozenset(weak_etags or ()) + self.star_tag = star_tag + + def as_set(self, include_weak: bool = False) -> set[str]: + """Convert the `ETags` object into a python set. Per default all the + weak etags are not part of this set.""" + rv = set(self._strong) + if include_weak: + rv.update(self._weak) + return rv + + def is_weak(self, etag: str) -> bool: + """Check if an etag is weak.""" + return etag in self._weak + + def is_strong(self, etag: str) -> bool: + """Check if an etag is strong.""" + return etag in self._strong + + def contains_weak(self, etag: str) -> bool: + """Check if an etag is part of the set including weak and strong tags.""" + return self.is_weak(etag) or self.contains(etag) + + def contains(self, etag: str) -> bool: + """Check if an etag is part of the set ignoring weak tags. + It is also possible to use the ``in`` operator. + """ + if self.star_tag: + return True + return self.is_strong(etag) + + def contains_raw(self, etag: str) -> bool: + """When passed a quoted tag it will check if this tag is part of the + set. If the tag is weak it is checked against weak and strong tags, + otherwise strong only.""" + from ..http import unquote_etag + + etag, weak = unquote_etag(etag) + if weak: + return self.contains_weak(etag) + return self.contains(etag) + + def to_header(self) -> str: + """Convert the etags set into a HTTP header string.""" + if self.star_tag: + return "*" + return ", ".join( + [f'"{x}"' for x in self._strong] + [f'W/"{x}"' for x in self._weak] + ) + + def __call__( + self, + etag: str | None = None, + data: bytes | None = None, + include_weak: bool = False, + ) -> bool: + if etag is None: + if data is None: + raise TypeError("'data' is required when 'etag' is not given.") + + from ..http import generate_etag + + etag = generate_etag(data) + if include_weak: + if etag in self._weak: + return True + return etag in self._strong + + def __bool__(self) -> bool: + return bool(self.star_tag or self._strong or self._weak) + + def __str__(self) -> str: + return self.to_header() + + def __len__(self) -> int: + return len(self._strong) + + def __iter__(self) -> cabc.Iterator[str]: + return iter(self._strong) + + def __contains__(self, etag: str) -> bool: # type: ignore[override] + return self.contains(etag) + + def __repr__(self) -> str: + return f"<{type(self).__name__} {str(self)!r}>" diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/file_storage.py b/lib/python3.12/site-packages/werkzeug/datastructures/file_storage.py new file mode 100644 index 0000000000000000000000000000000000000000..3a17f5125a53bf41de0cb9488503b35580dca316 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/file_storage.py @@ -0,0 +1,209 @@ +from __future__ import annotations + +import collections.abc as cabc +import mimetypes +import os +import typing as t +from io import BytesIO +from os import fsdecode +from os import fspath + +from .._internal import _plain_int +from .headers import Headers +from .structures import MultiDict + + +class FileStorage: + """The :class:`FileStorage` class is a thin wrapper over incoming files. + It is used by the request object to represent uploaded files. All the + attributes of the wrapper stream are proxied by the file storage so + it's possible to do ``storage.read()`` instead of the long form + ``storage.stream.read()``. + """ + + def __init__( + self, + stream: t.IO[bytes] | None = None, + filename: str | None = None, + name: str | None = None, + content_type: str | None = None, + content_length: int | None = None, + headers: Headers | None = None, + ): + self.name = name + self.stream = stream or BytesIO() + + # If no filename is provided, attempt to get the filename from + # the stream object. Python names special streams like + # ```` with angular brackets, skip these streams. + if filename is None: + filename = getattr(stream, "name", None) + + if filename is not None: + filename = fsdecode(filename) + + if filename and filename[0] == "<" and filename[-1] == ">": + filename = None + else: + filename = fsdecode(filename) + + self.filename = filename + + if headers is None: + headers = Headers() + self.headers = headers + if content_type is not None: + headers["Content-Type"] = content_type + if content_length is not None: + headers["Content-Length"] = str(content_length) + + def _parse_content_type(self) -> None: + if not hasattr(self, "_parsed_content_type"): + self._parsed_content_type = http.parse_options_header(self.content_type) + + @property + def content_type(self) -> str | None: + """The content-type sent in the header. Usually not available""" + return self.headers.get("content-type") + + @property + def content_length(self) -> int: + """The content-length sent in the header. Usually not available""" + if "content-length" in self.headers: + try: + return _plain_int(self.headers["content-length"]) + except ValueError: + pass + + return 0 + + @property + def mimetype(self) -> str: + """Like :attr:`content_type`, but without parameters (eg, without + charset, type etc.) and always lowercase. For example if the content + type is ``text/HTML; charset=utf-8`` the mimetype would be + ``'text/html'``. + + .. versionadded:: 0.7 + """ + self._parse_content_type() + return self._parsed_content_type[0].lower() + + @property + def mimetype_params(self) -> dict[str, str]: + """The mimetype parameters as dict. For example if the content + type is ``text/html; charset=utf-8`` the params would be + ``{'charset': 'utf-8'}``. + + .. versionadded:: 0.7 + """ + self._parse_content_type() + return self._parsed_content_type[1] + + def save( + self, dst: str | os.PathLike[str] | t.IO[bytes], buffer_size: int = 16384 + ) -> None: + """Save the file to a destination path or file object. If the + destination is a file object you have to close it yourself after the + call. The buffer size is the number of bytes held in memory during + the copy process. It defaults to 16KB. + + For secure file saving also have a look at :func:`secure_filename`. + + :param dst: a filename, :class:`os.PathLike`, or open file + object to write to. + :param buffer_size: Passed as the ``length`` parameter of + :func:`shutil.copyfileobj`. + + .. versionchanged:: 1.0 + Supports :mod:`pathlib`. + """ + from shutil import copyfileobj + + close_dst = False + + if hasattr(dst, "__fspath__"): + dst = fspath(dst) + + if isinstance(dst, str): + dst = open(dst, "wb") + close_dst = True + + try: + copyfileobj(self.stream, dst, buffer_size) + finally: + if close_dst: + dst.close() + + def close(self) -> None: + """Close the underlying file if possible.""" + try: + self.stream.close() + except Exception: + pass + + def __bool__(self) -> bool: + return bool(self.filename) + + def __getattr__(self, name: str) -> t.Any: + try: + return getattr(self.stream, name) + except AttributeError: + # SpooledTemporaryFile on Python < 3.11 doesn't implement IOBase, + # get the attribute from its backing file instead. + if hasattr(self.stream, "_file"): + return getattr(self.stream._file, name) + raise + + def __iter__(self) -> cabc.Iterator[bytes]: + return iter(self.stream) + + def __repr__(self) -> str: + return f"<{type(self).__name__}: {self.filename!r} ({self.content_type!r})>" + + +class FileMultiDict(MultiDict[str, FileStorage]): + """A special :class:`MultiDict` that has convenience methods to add + files to it. This is used for :class:`EnvironBuilder` and generally + useful for unittesting. + + .. versionadded:: 0.5 + """ + + def add_file( + self, + name: str, + file: str | os.PathLike[str] | t.IO[bytes] | FileStorage, + filename: str | None = None, + content_type: str | None = None, + ) -> None: + """Adds a new file to the dict. `file` can be a file name or + a :class:`file`-like or a :class:`FileStorage` object. + + :param name: the name of the field. + :param file: a filename or :class:`file`-like object + :param filename: an optional filename + :param content_type: an optional content type + """ + if isinstance(file, FileStorage): + self.add(name, file) + return + + if isinstance(file, (str, os.PathLike)): + if filename is None: + filename = os.fspath(file) + + file_obj: t.IO[bytes] = open(file, "rb") + else: + file_obj = file # type: ignore[assignment] + + if filename and content_type is None: + content_type = ( + mimetypes.guess_type(filename)[0] or "application/octet-stream" + ) + + self.add(name, FileStorage(file_obj, filename, name, content_type)) + + +# circular dependencies +from .. import http # noqa: E402 diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/headers.py b/lib/python3.12/site-packages/werkzeug/datastructures/headers.py new file mode 100644 index 0000000000000000000000000000000000000000..b8983622b8300f5c444f8351b95f29e5f319866d --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/headers.py @@ -0,0 +1,662 @@ +from __future__ import annotations + +import collections.abc as cabc +import re +import typing as t + +from .._internal import _missing +from ..exceptions import BadRequestKeyError +from .mixins import ImmutableHeadersMixin +from .structures import iter_multi_items +from .structures import MultiDict + +if t.TYPE_CHECKING: + import typing_extensions as te + from _typeshed.wsgi import WSGIEnvironment + +T = t.TypeVar("T") + + +class Headers: + """An object that stores some headers. It has a dict-like interface, + but is ordered, can store the same key multiple times, and iterating + yields ``(key, value)`` pairs instead of only keys. + + This data structure is useful if you want a nicer way to handle WSGI + headers which are stored as tuples in a list. + + From Werkzeug 0.3 onwards, the :exc:`KeyError` raised by this class is + also a subclass of the :class:`~exceptions.BadRequest` HTTP exception + and will render a page for a ``400 BAD REQUEST`` if caught in a + catch-all for HTTP exceptions. + + Headers is mostly compatible with the Python :class:`wsgiref.headers.Headers` + class, with the exception of `__getitem__`. :mod:`wsgiref` will return + `None` for ``headers['missing']``, whereas :class:`Headers` will raise + a :class:`KeyError`. + + To create a new ``Headers`` object, pass it a list, dict, or + other ``Headers`` object with default values. These values are + validated the same way values added later are. + + :param defaults: The list of default values for the :class:`Headers`. + + .. versionchanged:: 3.1 + Implement ``|`` and ``|=`` operators. + + .. versionchanged:: 2.1.0 + Default values are validated the same as values added later. + + .. versionchanged:: 0.9 + This data structure now stores unicode values similar to how the + multi dicts do it. The main difference is that bytes can be set as + well which will automatically be latin1 decoded. + + .. versionchanged:: 0.9 + The :meth:`linked` function was removed without replacement as it + was an API that does not support the changes to the encoding model. + """ + + def __init__( + self, + defaults: ( + Headers + | MultiDict[str, t.Any] + | cabc.Mapping[str, t.Any | list[t.Any] | tuple[t.Any, ...] | set[t.Any]] + | cabc.Iterable[tuple[str, t.Any]] + | None + ) = None, + ) -> None: + self._list: list[tuple[str, str]] = [] + + if defaults is not None: + self.extend(defaults) + + @t.overload + def __getitem__(self, key: str) -> str: ... + @t.overload + def __getitem__(self, key: int) -> tuple[str, str]: ... + @t.overload + def __getitem__(self, key: slice) -> te.Self: ... + def __getitem__(self, key: str | int | slice) -> str | tuple[str, str] | te.Self: + if isinstance(key, str): + return self._get_key(key) + + if isinstance(key, int): + return self._list[key] + + return self.__class__(self._list[key]) + + def _get_key(self, key: str) -> str: + ikey = key.lower() + + for k, v in self._list: + if k.lower() == ikey: + return v + + raise BadRequestKeyError(key) + + def __eq__(self, other: object) -> bool: + if other.__class__ is not self.__class__: + return NotImplemented + + def lowered(item: tuple[str, ...]) -> tuple[str, ...]: + return item[0].lower(), *item[1:] + + return set(map(lowered, other._list)) == set(map(lowered, self._list)) # type: ignore[attr-defined] + + __hash__ = None # type: ignore[assignment] + + @t.overload + def get(self, key: str) -> str | None: ... + @t.overload + def get(self, key: str, default: str) -> str: ... + @t.overload + def get(self, key: str, default: T) -> str | T: ... + @t.overload + def get(self, key: str, type: cabc.Callable[[str], T]) -> T | None: ... + @t.overload + def get(self, key: str, default: T, type: cabc.Callable[[str], T]) -> T: ... + def get( # type: ignore[misc] + self, + key: str, + default: str | T | None = None, + type: cabc.Callable[[str], T] | None = None, + ) -> str | T | None: + """Return the default value if the requested data doesn't exist. + If `type` is provided and is a callable it should convert the value, + return it or raise a :exc:`ValueError` if that is not possible. In + this case the function will return the default as if the value was not + found: + + >>> d = Headers([('Content-Length', '42')]) + >>> d.get('Content-Length', type=int) + 42 + + :param key: The key to be looked up. + :param default: The default value to be returned if the key can't + be looked up. If not further specified `None` is + returned. + :param type: A callable that is used to cast the value in the + :class:`Headers`. If a :exc:`ValueError` is raised + by this callable the default value is returned. + + .. versionchanged:: 3.0 + The ``as_bytes`` parameter was removed. + + .. versionchanged:: 0.9 + The ``as_bytes`` parameter was added. + """ + try: + rv = self._get_key(key) + except KeyError: + return default + + if type is None: + return rv + + try: + return type(rv) + except ValueError: + return default + + @t.overload + def getlist(self, key: str) -> list[str]: ... + @t.overload + def getlist(self, key: str, type: cabc.Callable[[str], T]) -> list[T]: ... + def getlist( + self, key: str, type: cabc.Callable[[str], T] | None = None + ) -> list[str] | list[T]: + """Return the list of items for a given key. If that key is not in the + :class:`Headers`, the return value will be an empty list. Just like + :meth:`get`, :meth:`getlist` accepts a `type` parameter. All items will + be converted with the callable defined there. + + :param key: The key to be looked up. + :param type: A callable that is used to cast the value in the + :class:`Headers`. If a :exc:`ValueError` is raised + by this callable the value will be removed from the list. + :return: a :class:`list` of all the values for the key. + + .. versionchanged:: 3.0 + The ``as_bytes`` parameter was removed. + + .. versionchanged:: 0.9 + The ``as_bytes`` parameter was added. + """ + ikey = key.lower() + + if type is not None: + result = [] + + for k, v in self: + if k.lower() == ikey: + try: + result.append(type(v)) + except ValueError: + continue + + return result + + return [v for k, v in self if k.lower() == ikey] + + def get_all(self, name: str) -> list[str]: + """Return a list of all the values for the named field. + + This method is compatible with the :mod:`wsgiref` + :meth:`~wsgiref.headers.Headers.get_all` method. + """ + return self.getlist(name) + + def items(self, lower: bool = False) -> t.Iterable[tuple[str, str]]: + for key, value in self: + if lower: + key = key.lower() + yield key, value + + def keys(self, lower: bool = False) -> t.Iterable[str]: + for key, _ in self.items(lower): + yield key + + def values(self) -> t.Iterable[str]: + for _, value in self.items(): + yield value + + def extend( + self, + arg: ( + Headers + | MultiDict[str, t.Any] + | cabc.Mapping[str, t.Any | list[t.Any] | tuple[t.Any, ...] | set[t.Any]] + | cabc.Iterable[tuple[str, t.Any]] + | None + ) = None, + /, + **kwargs: str, + ) -> None: + """Extend headers in this object with items from another object + containing header items as well as keyword arguments. + + To replace existing keys instead of extending, use + :meth:`update` instead. + + If provided, the first argument can be another :class:`Headers` + object, a :class:`MultiDict`, :class:`dict`, or iterable of + pairs. + + .. versionchanged:: 1.0 + Support :class:`MultiDict`. Allow passing ``kwargs``. + """ + if arg is not None: + for key, value in iter_multi_items(arg): + self.add(key, value) + + for key, value in iter_multi_items(kwargs): + self.add(key, value) + + def __delitem__(self, key: str | int | slice) -> None: + if isinstance(key, str): + self._del_key(key) + return + + del self._list[key] + + def _del_key(self, key: str) -> None: + key = key.lower() + new = [] + + for k, v in self._list: + if k.lower() != key: + new.append((k, v)) + + self._list[:] = new + + def remove(self, key: str) -> None: + """Remove a key. + + :param key: The key to be removed. + """ + return self._del_key(key) + + @t.overload + def pop(self) -> tuple[str, str]: ... + @t.overload + def pop(self, key: str) -> str: ... + @t.overload + def pop(self, key: int | None = ...) -> tuple[str, str]: ... + @t.overload + def pop(self, key: str, default: str) -> str: ... + @t.overload + def pop(self, key: str, default: T) -> str | T: ... + def pop( + self, + key: str | int | None = None, + default: str | T = _missing, # type: ignore[assignment] + ) -> str | tuple[str, str] | T: + """Removes and returns a key or index. + + :param key: The key to be popped. If this is an integer the item at + that position is removed, if it's a string the value for + that key is. If the key is omitted or `None` the last + item is removed. + :return: an item. + """ + if key is None: + return self._list.pop() + + if isinstance(key, int): + return self._list.pop(key) + + try: + rv = self._get_key(key) + except KeyError: + if default is not _missing: + return default + + raise + + self.remove(key) + return rv + + def popitem(self) -> tuple[str, str]: + """Removes a key or index and returns a (key, value) item.""" + return self._list.pop() + + def __contains__(self, key: str) -> bool: + """Check if a key is present.""" + try: + self._get_key(key) + except KeyError: + return False + + return True + + def __iter__(self) -> t.Iterator[tuple[str, str]]: + """Yield ``(key, value)`` tuples.""" + return iter(self._list) + + def __len__(self) -> int: + return len(self._list) + + def add(self, key: str, value: t.Any, /, **kwargs: t.Any) -> None: + """Add a new header tuple to the list. + + Keyword arguments can specify additional parameters for the header + value, with underscores converted to dashes:: + + >>> d = Headers() + >>> d.add('Content-Type', 'text/plain') + >>> d.add('Content-Disposition', 'attachment', filename='foo.png') + + The keyword argument dumping uses :func:`dump_options_header` + behind the scenes. + + .. versionchanged:: 0.4.1 + keyword arguments were added for :mod:`wsgiref` compatibility. + """ + if kwargs: + value = _options_header_vkw(value, kwargs) + + value_str = _str_header_value(value) + self._list.append((key, value_str)) + + def add_header(self, key: str, value: t.Any, /, **kwargs: t.Any) -> None: + """Add a new header tuple to the list. + + An alias for :meth:`add` for compatibility with the :mod:`wsgiref` + :meth:`~wsgiref.headers.Headers.add_header` method. + """ + self.add(key, value, **kwargs) + + def clear(self) -> None: + """Clears all headers.""" + self._list.clear() + + def set(self, key: str, value: t.Any, /, **kwargs: t.Any) -> None: + """Remove all header tuples for `key` and add a new one. The newly + added key either appears at the end of the list if there was no + entry or replaces the first one. + + Keyword arguments can specify additional parameters for the header + value, with underscores converted to dashes. See :meth:`add` for + more information. + + .. versionchanged:: 0.6.1 + :meth:`set` now accepts the same arguments as :meth:`add`. + + :param key: The key to be inserted. + :param value: The value to be inserted. + """ + if kwargs: + value = _options_header_vkw(value, kwargs) + + value_str = _str_header_value(value) + + if not self._list: + self._list.append((key, value_str)) + return + + iter_list = iter(self._list) + ikey = key.lower() + + for idx, (old_key, _) in enumerate(iter_list): + if old_key.lower() == ikey: + # replace first occurrence + self._list[idx] = (key, value_str) + break + else: + # no existing occurrences + self._list.append((key, value_str)) + return + + # remove remaining occurrences + self._list[idx + 1 :] = [t for t in iter_list if t[0].lower() != ikey] + + def setlist(self, key: str, values: cabc.Iterable[t.Any]) -> None: + """Remove any existing values for a header and add new ones. + + :param key: The header key to set. + :param values: An iterable of values to set for the key. + + .. versionadded:: 1.0 + """ + if values: + values_iter = iter(values) + self.set(key, next(values_iter)) + + for value in values_iter: + self.add(key, value) + else: + self.remove(key) + + def setdefault(self, key: str, default: t.Any) -> str: + """Return the first value for the key if it is in the headers, + otherwise set the header to the value given by ``default`` and + return that. + + :param key: The header key to get. + :param default: The value to set for the key if it is not in the + headers. + """ + try: + return self._get_key(key) + except KeyError: + pass + + self.set(key, default) + return self._get_key(key) + + def setlistdefault(self, key: str, default: cabc.Iterable[t.Any]) -> list[str]: + """Return the list of values for the key if it is in the + headers, otherwise set the header to the list of values given + by ``default`` and return that. + + Unlike :meth:`MultiDict.setlistdefault`, modifying the returned + list will not affect the headers. + + :param key: The header key to get. + :param default: An iterable of values to set for the key if it + is not in the headers. + + .. versionadded:: 1.0 + """ + if key not in self: + self.setlist(key, default) + + return self.getlist(key) + + @t.overload + def __setitem__(self, key: str, value: t.Any) -> None: ... + @t.overload + def __setitem__(self, key: int, value: tuple[str, t.Any]) -> None: ... + @t.overload + def __setitem__( + self, key: slice, value: cabc.Iterable[tuple[str, t.Any]] + ) -> None: ... + def __setitem__( + self, + key: str | int | slice, + value: t.Any | tuple[str, t.Any] | cabc.Iterable[tuple[str, t.Any]], + ) -> None: + """Like :meth:`set` but also supports index/slice based setting.""" + if isinstance(key, str): + self.set(key, value) + elif isinstance(key, int): + self._list[key] = value[0], _str_header_value(value[1]) # type: ignore[index] + else: + self._list[key] = [(k, _str_header_value(v)) for k, v in value] # type: ignore[misc] + + def update( + self, + arg: ( + Headers + | MultiDict[str, t.Any] + | cabc.Mapping[ + str, t.Any | list[t.Any] | tuple[t.Any, ...] | cabc.Set[t.Any] + ] + | cabc.Iterable[tuple[str, t.Any]] + | None + ) = None, + /, + **kwargs: t.Any | list[t.Any] | tuple[t.Any, ...] | cabc.Set[t.Any], + ) -> None: + """Replace headers in this object with items from another + headers object and keyword arguments. + + To extend existing keys instead of replacing, use :meth:`extend` + instead. + + If provided, the first argument can be another :class:`Headers` + object, a :class:`MultiDict`, :class:`dict`, or iterable of + pairs. + + .. versionadded:: 1.0 + """ + if arg is not None: + if isinstance(arg, (Headers, MultiDict)): + for key in arg.keys(): + self.setlist(key, arg.getlist(key)) + elif isinstance(arg, cabc.Mapping): + for key, value in arg.items(): + if isinstance(value, (list, tuple, set)): + self.setlist(key, value) + else: + self.set(key, value) + else: + for key, value in arg: + self.set(key, value) + + for key, value in kwargs.items(): + if isinstance(value, (list, tuple, set)): + self.setlist(key, value) + else: + self.set(key, value) + + def __or__( + self, + other: cabc.Mapping[ + str, t.Any | list[t.Any] | tuple[t.Any, ...] | cabc.Set[t.Any] + ], + ) -> te.Self: + if not isinstance(other, cabc.Mapping): + return NotImplemented + + rv = self.copy() + rv.update(other) + return rv + + def __ior__( + self, + other: ( + cabc.Mapping[str, t.Any | list[t.Any] | tuple[t.Any, ...] | cabc.Set[t.Any]] + | cabc.Iterable[tuple[str, t.Any]] + ), + ) -> te.Self: + if not isinstance(other, (cabc.Mapping, cabc.Iterable)): + return NotImplemented + + self.update(other) + return self + + def to_wsgi_list(self) -> list[tuple[str, str]]: + """Convert the headers into a list suitable for WSGI. + + :return: list + """ + return list(self) + + def copy(self) -> te.Self: + return self.__class__(self._list) + + def __copy__(self) -> te.Self: + return self.copy() + + def __str__(self) -> str: + """Returns formatted headers suitable for HTTP transmission.""" + strs = [] + for key, value in self.to_wsgi_list(): + strs.append(f"{key}: {value}") + strs.append("\r\n") + return "\r\n".join(strs) + + def __repr__(self) -> str: + return f"{type(self).__name__}({list(self)!r})" + + +def _options_header_vkw(value: str, kw: dict[str, t.Any]) -> str: + return http.dump_options_header( + value, {k.replace("_", "-"): v for k, v in kw.items()} + ) + + +_newline_re = re.compile(r"[\r\n]") + + +def _str_header_value(value: t.Any) -> str: + if not isinstance(value, str): + value = str(value) + + if _newline_re.search(value) is not None: + raise ValueError("Header values must not contain newline characters.") + + return value # type: ignore[no-any-return] + + +class EnvironHeaders(ImmutableHeadersMixin, Headers): # type: ignore[misc] + """Read only version of the headers from a WSGI environment. This + provides the same interface as `Headers` and is constructed from + a WSGI environment. + From Werkzeug 0.3 onwards, the `KeyError` raised by this class is also a + subclass of the :exc:`~exceptions.BadRequest` HTTP exception and will + render a page for a ``400 BAD REQUEST`` if caught in a catch-all for + HTTP exceptions. + """ + + def __init__(self, environ: WSGIEnvironment) -> None: + super().__init__() + self.environ = environ + + def __eq__(self, other: object) -> bool: + if not isinstance(other, EnvironHeaders): + return NotImplemented + + return self.environ is other.environ + + __hash__ = None + + def __getitem__(self, key: str) -> str: # type: ignore[override] + return self._get_key(key) + + def _get_key(self, key: str) -> str: + if not isinstance(key, str): + raise BadRequestKeyError(key) + + key = key.upper().replace("-", "_") + + if key in {"CONTENT_TYPE", "CONTENT_LENGTH"}: + return self.environ[key] # type: ignore[no-any-return] + + return self.environ[f"HTTP_{key}"] # type: ignore[no-any-return] + + def __len__(self) -> int: + return sum(1 for _ in self) + + def __iter__(self) -> cabc.Iterator[tuple[str, str]]: + for key, value in self.environ.items(): + if key.startswith("HTTP_") and key not in { + "HTTP_CONTENT_TYPE", + "HTTP_CONTENT_LENGTH", + }: + yield key[5:].replace("_", "-").title(), value + elif key in {"CONTENT_TYPE", "CONTENT_LENGTH"} and value: + yield key.replace("_", "-").title(), value + + def copy(self) -> t.NoReturn: + raise TypeError(f"cannot create {type(self).__name__!r} copies") + + def __or__(self, other: t.Any) -> t.NoReturn: + raise TypeError(f"cannot create {type(self).__name__!r} copies") + + +# circular dependencies +from .. import http # noqa: E402 diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/mixins.py b/lib/python3.12/site-packages/werkzeug/datastructures/mixins.py new file mode 100644 index 0000000000000000000000000000000000000000..8e00bdadd30e8eab9a7f18640c68ad96023fd6b9 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/mixins.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import collections.abc as cabc +import typing as t +from functools import update_wrapper +from itertools import repeat + +from .._internal import _missing + +if t.TYPE_CHECKING: + import typing_extensions as te + +K = t.TypeVar("K") +V = t.TypeVar("V") +T = t.TypeVar("T") +F = t.TypeVar("F", bound=cabc.Callable[..., t.Any]) + + +def _immutable_error(self: t.Any) -> t.NoReturn: + raise TypeError(f"{type(self).__name__!r} objects are immutable") + + +class ImmutableListMixin: + """Makes a :class:`list` immutable. + + .. versionadded:: 0.5 + + :private: + """ + + _hash_cache: int | None = None + + def __hash__(self) -> int: + if self._hash_cache is not None: + return self._hash_cache + rv = self._hash_cache = hash(tuple(self)) # type: ignore[arg-type] + return rv + + def __reduce_ex__(self, protocol: t.SupportsIndex) -> t.Any: + return type(self), (list(self),) # type: ignore[call-overload] + + def __delitem__(self, key: t.Any) -> t.NoReturn: + _immutable_error(self) + + def __iadd__(self, other: t.Any) -> t.NoReturn: + _immutable_error(self) + + def __imul__(self, other: t.Any) -> t.NoReturn: + _immutable_error(self) + + def __setitem__(self, key: t.Any, value: t.Any) -> t.NoReturn: + _immutable_error(self) + + def append(self, item: t.Any) -> t.NoReturn: + _immutable_error(self) + + def remove(self, item: t.Any) -> t.NoReturn: + _immutable_error(self) + + def extend(self, iterable: t.Any) -> t.NoReturn: + _immutable_error(self) + + def insert(self, pos: t.Any, value: t.Any) -> t.NoReturn: + _immutable_error(self) + + def pop(self, index: t.Any = -1) -> t.NoReturn: + _immutable_error(self) + + def reverse(self: t.Any) -> t.NoReturn: + _immutable_error(self) + + def sort(self, key: t.Any = None, reverse: t.Any = False) -> t.NoReturn: + _immutable_error(self) + + +class ImmutableDictMixin(t.Generic[K, V]): + """Makes a :class:`dict` immutable. + + .. versionchanged:: 3.1 + Disallow ``|=`` operator. + + .. versionadded:: 0.5 + + :private: + """ + + _hash_cache: int | None = None + + @classmethod + @t.overload + def fromkeys( + cls, keys: cabc.Iterable[K], value: None + ) -> ImmutableDictMixin[K, t.Any | None]: ... + @classmethod + @t.overload + def fromkeys(cls, keys: cabc.Iterable[K], value: V) -> ImmutableDictMixin[K, V]: ... + @classmethod + def fromkeys( + cls, keys: cabc.Iterable[K], value: V | None = None + ) -> ImmutableDictMixin[K, t.Any | None] | ImmutableDictMixin[K, V]: + instance = super().__new__(cls) + instance.__init__(zip(keys, repeat(value))) # type: ignore[misc] + return instance + + def __reduce_ex__(self, protocol: t.SupportsIndex) -> t.Any: + return type(self), (dict(self),) # type: ignore[call-overload] + + def _iter_hashitems(self) -> t.Iterable[t.Any]: + return self.items() # type: ignore[attr-defined,no-any-return] + + def __hash__(self) -> int: + if self._hash_cache is not None: + return self._hash_cache + rv = self._hash_cache = hash(frozenset(self._iter_hashitems())) + return rv + + def setdefault(self, key: t.Any, default: t.Any = None) -> t.NoReturn: + _immutable_error(self) + + def update(self, arg: t.Any, /, **kwargs: t.Any) -> t.NoReturn: + _immutable_error(self) + + def __ior__(self, other: t.Any) -> t.NoReturn: + _immutable_error(self) + + def pop(self, key: t.Any, default: t.Any = None) -> t.NoReturn: + _immutable_error(self) + + def popitem(self) -> t.NoReturn: + _immutable_error(self) + + def __setitem__(self, key: t.Any, value: t.Any) -> t.NoReturn: + _immutable_error(self) + + def __delitem__(self, key: t.Any) -> t.NoReturn: + _immutable_error(self) + + def clear(self) -> t.NoReturn: + _immutable_error(self) + + +class ImmutableMultiDictMixin(ImmutableDictMixin[K, V]): + """Makes a :class:`MultiDict` immutable. + + .. versionadded:: 0.5 + + :private: + """ + + def __reduce_ex__(self, protocol: t.SupportsIndex) -> t.Any: + return type(self), (list(self.items(multi=True)),) # type: ignore[attr-defined] + + def _iter_hashitems(self) -> t.Iterable[t.Any]: + return self.items(multi=True) # type: ignore[attr-defined,no-any-return] + + def add(self, key: t.Any, value: t.Any) -> t.NoReturn: + _immutable_error(self) + + def popitemlist(self) -> t.NoReturn: + _immutable_error(self) + + def poplist(self, key: t.Any) -> t.NoReturn: + _immutable_error(self) + + def setlist(self, key: t.Any, new_list: t.Any) -> t.NoReturn: + _immutable_error(self) + + def setlistdefault(self, key: t.Any, default_list: t.Any = None) -> t.NoReturn: + _immutable_error(self) + + +class ImmutableHeadersMixin: + """Makes a :class:`Headers` immutable. We do not mark them as + hashable though since the only usecase for this datastructure + in Werkzeug is a view on a mutable structure. + + .. versionchanged:: 3.1 + Disallow ``|=`` operator. + + .. versionadded:: 0.5 + + :private: + """ + + def __delitem__(self, key: t.Any, **kwargs: t.Any) -> t.NoReturn: + _immutable_error(self) + + def __setitem__(self, key: t.Any, value: t.Any) -> t.NoReturn: + _immutable_error(self) + + def set(self, key: t.Any, value: t.Any, /, **kwargs: t.Any) -> t.NoReturn: + _immutable_error(self) + + def setlist(self, key: t.Any, values: t.Any) -> t.NoReturn: + _immutable_error(self) + + def add(self, key: t.Any, value: t.Any, /, **kwargs: t.Any) -> t.NoReturn: + _immutable_error(self) + + def add_header(self, key: t.Any, value: t.Any, /, **kwargs: t.Any) -> t.NoReturn: + _immutable_error(self) + + def remove(self, key: t.Any) -> t.NoReturn: + _immutable_error(self) + + def extend(self, arg: t.Any, /, **kwargs: t.Any) -> t.NoReturn: + _immutable_error(self) + + def update(self, arg: t.Any, /, **kwargs: t.Any) -> t.NoReturn: + _immutable_error(self) + + def __ior__(self, other: t.Any) -> t.NoReturn: + _immutable_error(self) + + def insert(self, pos: t.Any, value: t.Any) -> t.NoReturn: + _immutable_error(self) + + def pop(self, key: t.Any = None, default: t.Any = _missing) -> t.NoReturn: + _immutable_error(self) + + def popitem(self) -> t.NoReturn: + _immutable_error(self) + + def setdefault(self, key: t.Any, default: t.Any) -> t.NoReturn: + _immutable_error(self) + + def setlistdefault(self, key: t.Any, default: t.Any) -> t.NoReturn: + _immutable_error(self) + + +def _always_update(f: F) -> F: + def wrapper( + self: UpdateDictMixin[t.Any, t.Any], /, *args: t.Any, **kwargs: t.Any + ) -> t.Any: + rv = f(self, *args, **kwargs) + + if self.on_update is not None: + self.on_update(self) + + return rv + + return update_wrapper(wrapper, f) # type: ignore[return-value] + + +class UpdateDictMixin(dict[K, V]): + """Makes dicts call `self.on_update` on modifications. + + .. versionchanged:: 3.1 + Implement ``|=`` operator. + + .. versionadded:: 0.5 + + :private: + """ + + on_update: cabc.Callable[[te.Self], None] | None = None + + def setdefault(self: te.Self, key: K, default: V | None = None) -> V: + modified = key not in self + rv = super().setdefault(key, default) # type: ignore[arg-type] + if modified and self.on_update is not None: + self.on_update(self) + return rv + + @t.overload + def pop(self: te.Self, key: K) -> V: ... + @t.overload + def pop(self: te.Self, key: K, default: V) -> V: ... + @t.overload + def pop(self: te.Self, key: K, default: T) -> T: ... + def pop( + self: te.Self, + key: K, + default: V | T = _missing, # type: ignore[assignment] + ) -> V | T: + modified = key in self + if default is _missing: + rv: V | T = super().pop(key) + else: + rv = super().pop(key, default) + if modified and self.on_update is not None: + self.on_update(self) + return rv + + @_always_update + def __setitem__(self, key: K, value: V) -> None: + super().__setitem__(key, value) + + @_always_update + def __delitem__(self, key: K) -> None: + super().__delitem__(key) + + @_always_update + def clear(self) -> None: + super().clear() + + @_always_update + def popitem(self) -> tuple[K, V]: + return super().popitem() + + @_always_update + def update( # type: ignore[override] + self, + arg: cabc.Mapping[K, V] | cabc.Iterable[tuple[K, V]] | None = None, + /, + **kwargs: V, + ) -> None: + if arg is None: + super().update(**kwargs) # type: ignore[call-overload] + else: + super().update(arg, **kwargs) + + @_always_update + def __ior__( # type: ignore[override] + self, other: cabc.Mapping[K, V] | cabc.Iterable[tuple[K, V]] + ) -> te.Self: + return super().__ior__(other) diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/range.py b/lib/python3.12/site-packages/werkzeug/datastructures/range.py new file mode 100644 index 0000000000000000000000000000000000000000..72f863d2c97ff8e7e62540d2f839f92e404cbf3b --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/range.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import collections.abc as cabc +import typing as t +from datetime import datetime + +if t.TYPE_CHECKING: + import typing_extensions as te + +T = t.TypeVar("T") + + +class IfRange: + """Very simple object that represents the `If-Range` header in parsed + form. It will either have neither a etag or date or one of either but + never both. + + .. versionadded:: 0.7 + """ + + def __init__(self, etag: str | None = None, date: datetime | None = None): + #: The etag parsed and unquoted. Ranges always operate on strong + #: etags so the weakness information is not necessary. + self.etag = etag + #: The date in parsed format or `None`. + self.date = date + + def to_header(self) -> str: + """Converts the object back into an HTTP header.""" + if self.date is not None: + return http.http_date(self.date) + if self.etag is not None: + return http.quote_etag(self.etag) + return "" + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + return f"<{type(self).__name__} {str(self)!r}>" + + +class Range: + """Represents a ``Range`` header. All methods only support only + bytes as the unit. Stores a list of ranges if given, but the methods + only work if only one range is provided. + + :raise ValueError: If the ranges provided are invalid. + + .. versionchanged:: 0.15 + The ranges passed in are validated. + + .. versionadded:: 0.7 + """ + + def __init__( + self, units: str, ranges: cabc.Sequence[tuple[int, int | None]] + ) -> None: + #: The units of this range. Usually "bytes". + self.units = units + #: A list of ``(begin, end)`` tuples for the range header provided. + #: The ranges are non-inclusive. + self.ranges = ranges + + for start, end in ranges: + if start is None or (end is not None and (start < 0 or start >= end)): + raise ValueError(f"{(start, end)} is not a valid range.") + + def range_for_length(self, length: int | None) -> tuple[int, int] | None: + """If the range is for bytes, the length is not None and there is + exactly one range and it is satisfiable it returns a ``(start, stop)`` + tuple, otherwise `None`. + """ + if self.units != "bytes" or length is None or len(self.ranges) != 1: + return None + start, end = self.ranges[0] + if end is None: + end = length + if start < 0: + start += length + if http.is_byte_range_valid(start, end, length): + return start, min(end, length) + return None + + def make_content_range(self, length: int | None) -> ContentRange | None: + """Creates a :class:`~werkzeug.datastructures.ContentRange` object + from the current range and given content length. + """ + rng = self.range_for_length(length) + if rng is not None: + return ContentRange(self.units, rng[0], rng[1], length) + return None + + def to_header(self) -> str: + """Converts the object back into an HTTP header.""" + ranges = [] + for begin, end in self.ranges: + if end is None: + ranges.append(f"{begin}-" if begin >= 0 else str(begin)) + else: + ranges.append(f"{begin}-{end - 1}") + return f"{self.units}={','.join(ranges)}" + + def to_content_range_header(self, length: int | None) -> str | None: + """Converts the object into `Content-Range` HTTP header, + based on given length + """ + range = self.range_for_length(length) + if range is not None: + return f"{self.units} {range[0]}-{range[1] - 1}/{length}" + return None + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + return f"<{type(self).__name__} {str(self)!r}>" + + +class _CallbackProperty(t.Generic[T]): + def __set_name__(self, owner: type[ContentRange], name: str) -> None: + self.attr = f"_{name}" + + @t.overload + def __get__(self, instance: None, owner: None) -> te.Self: ... + @t.overload + def __get__(self, instance: ContentRange, owner: type[ContentRange]) -> T: ... + def __get__( + self, instance: ContentRange | None, owner: type[ContentRange] | None + ) -> te.Self | T: + if instance is None: + return self + + return instance.__dict__[self.attr] # type: ignore[no-any-return] + + def __set__(self, instance: ContentRange, value: T) -> None: + instance.__dict__[self.attr] = value + + if instance.on_update is not None: + instance.on_update(instance) + + +class ContentRange: + """Represents the content range header. + + .. versionadded:: 0.7 + """ + + def __init__( + self, + units: str | None, + start: int | None, + stop: int | None, + length: int | None = None, + on_update: cabc.Callable[[ContentRange], None] | None = None, + ) -> None: + self.on_update = on_update + self.set(start, stop, length, units) + + #: The units to use, usually "bytes" + units: str | None = _CallbackProperty() # type: ignore[assignment] + #: The start point of the range or `None`. + start: int | None = _CallbackProperty() # type: ignore[assignment] + #: The stop point of the range (non-inclusive) or `None`. Can only be + #: `None` if also start is `None`. + stop: int | None = _CallbackProperty() # type: ignore[assignment] + #: The length of the range or `None`. + length: int | None = _CallbackProperty() # type: ignore[assignment] + + def set( + self, + start: int | None, + stop: int | None, + length: int | None = None, + units: str | None = "bytes", + ) -> None: + """Simple method to update the ranges.""" + assert http.is_byte_range_valid(start, stop, length), "Bad range provided" + self._units: str | None = units + self._start: int | None = start + self._stop: int | None = stop + self._length: int | None = length + if self.on_update is not None: + self.on_update(self) + + def unset(self) -> None: + """Sets the units to `None` which indicates that the header should + no longer be used. + """ + self.set(None, None, units=None) + + def to_header(self) -> str: + if self._units is None: + return "" + if self._length is None: + length: str | int = "*" + else: + length = self._length + if self._start is None: + return f"{self._units} */{length}" + return f"{self._units} {self._start}-{self._stop - 1}/{length}" # type: ignore[operator] + + def __bool__(self) -> bool: + return self._units is not None + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + return f"<{type(self).__name__} {str(self)!r}>" + + +# circular dependencies +from .. import http # noqa: E402 diff --git a/lib/python3.12/site-packages/werkzeug/datastructures/structures.py b/lib/python3.12/site-packages/werkzeug/datastructures/structures.py new file mode 100644 index 0000000000000000000000000000000000000000..8dc6437a6973fc5c17c19e423f18b0b21500d476 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/datastructures/structures.py @@ -0,0 +1,1239 @@ +from __future__ import annotations + +import collections.abc as cabc +import typing as t +from copy import deepcopy + +from .. import exceptions +from .._internal import _missing +from .mixins import ImmutableDictMixin +from .mixins import ImmutableListMixin +from .mixins import ImmutableMultiDictMixin +from .mixins import UpdateDictMixin + +if t.TYPE_CHECKING: + import typing_extensions as te + +K = t.TypeVar("K") +V = t.TypeVar("V") +T = t.TypeVar("T") + + +def iter_multi_items( + mapping: ( + MultiDict[K, V] + | cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + | cabc.Iterable[tuple[K, V]] + ), +) -> cabc.Iterator[tuple[K, V]]: + """Iterates over the items of a mapping yielding keys and values + without dropping any from more complex structures. + """ + if isinstance(mapping, MultiDict): + yield from mapping.items(multi=True) + elif isinstance(mapping, cabc.Mapping): + for key, value in mapping.items(): + if isinstance(value, (list, tuple, set)): + for v in value: + yield key, v + else: + yield key, value + else: + yield from mapping + + +class ImmutableList(ImmutableListMixin, list[V]): # type: ignore[misc] + """An immutable :class:`list`. + + .. versionadded:: 0.5 + + :private: + """ + + def __repr__(self) -> str: + return f"{type(self).__name__}({list.__repr__(self)})" + + +class TypeConversionDict(dict[K, V]): + """Works like a regular dict but the :meth:`get` method can perform + type conversions. :class:`MultiDict` and :class:`CombinedMultiDict` + are subclasses of this class and provide the same feature. + + .. versionadded:: 0.5 + """ + + @t.overload # type: ignore[override] + def get(self, key: K) -> V | None: ... + @t.overload + def get(self, key: K, default: V) -> V: ... + @t.overload + def get(self, key: K, default: T) -> V | T: ... + @t.overload + def get(self, key: str, type: cabc.Callable[[V], T]) -> T | None: ... + @t.overload + def get(self, key: str, default: T, type: cabc.Callable[[V], T]) -> T: ... + def get( # type: ignore[misc] + self, + key: K, + default: V | T | None = None, + type: cabc.Callable[[V], T] | None = None, + ) -> V | T | None: + """Return the default value if the requested data doesn't exist. + If `type` is provided and is a callable it should convert the value, + return it or raise a :exc:`ValueError` if that is not possible. In + this case the function will return the default as if the value was not + found: + + >>> d = TypeConversionDict(foo='42', bar='blub') + >>> d.get('foo', type=int) + 42 + >>> d.get('bar', -1, type=int) + -1 + + :param key: The key to be looked up. + :param default: The default value to be returned if the key can't + be looked up. If not further specified `None` is + returned. + :param type: A callable that is used to cast the value in the + :class:`MultiDict`. If a :exc:`ValueError` or a + :exc:`TypeError` is raised by this callable the default + value is returned. + + .. versionchanged:: 3.0.2 + Returns the default value on :exc:`TypeError`, too. + """ + try: + rv = self[key] + except KeyError: + return default + + if type is None: + return rv + + try: + return type(rv) + except (ValueError, TypeError): + return default + + +class ImmutableTypeConversionDict(ImmutableDictMixin[K, V], TypeConversionDict[K, V]): # type: ignore[misc] + """Works like a :class:`TypeConversionDict` but does not support + modifications. + + .. versionadded:: 0.5 + """ + + def copy(self) -> TypeConversionDict[K, V]: + """Return a shallow mutable copy of this object. Keep in mind that + the standard library's :func:`copy` function is a no-op for this class + like for any other python immutable type (eg: :class:`tuple`). + """ + return TypeConversionDict(self) + + def __copy__(self) -> te.Self: + return self + + +class MultiDict(TypeConversionDict[K, V]): + """A :class:`MultiDict` is a dictionary subclass customized to deal with + multiple values for the same key which is for example used by the parsing + functions in the wrappers. This is necessary because some HTML form + elements pass multiple values for the same key. + + :class:`MultiDict` implements all standard dictionary methods. + Internally, it saves all values for a key as a list, but the standard dict + access methods will only return the first value for a key. If you want to + gain access to the other values, too, you have to use the `list` methods as + explained below. + + Basic Usage: + + >>> d = MultiDict([('a', 'b'), ('a', 'c')]) + >>> d + MultiDict([('a', 'b'), ('a', 'c')]) + >>> d['a'] + 'b' + >>> d.getlist('a') + ['b', 'c'] + >>> 'a' in d + True + + It behaves like a normal dict thus all dict functions will only return the + first value when multiple values for one key are found. + + From Werkzeug 0.3 onwards, the `KeyError` raised by this class is also a + subclass of the :exc:`~exceptions.BadRequest` HTTP exception and will + render a page for a ``400 BAD REQUEST`` if caught in a catch-all for HTTP + exceptions. + + A :class:`MultiDict` can be constructed from an iterable of + ``(key, value)`` tuples, a dict, a :class:`MultiDict` or from Werkzeug 0.2 + onwards some keyword parameters. + + :param mapping: the initial value for the :class:`MultiDict`. Either a + regular dict, an iterable of ``(key, value)`` tuples + or `None`. + + .. versionchanged:: 3.1 + Implement ``|`` and ``|=`` operators. + """ + + def __init__( + self, + mapping: ( + MultiDict[K, V] + | cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + | cabc.Iterable[tuple[K, V]] + | None + ) = None, + ) -> None: + if mapping is None: + super().__init__() + elif isinstance(mapping, MultiDict): + super().__init__((k, vs[:]) for k, vs in mapping.lists()) # type: ignore[misc] + elif isinstance(mapping, cabc.Mapping): + tmp = {} + for key, value in mapping.items(): + if isinstance(value, (list, tuple, set)): + value = list(value) + + if not value: + continue + else: + value = [value] + tmp[key] = value + super().__init__(tmp) # type: ignore[arg-type] + else: + tmp = {} + for key, value in mapping: + tmp.setdefault(key, []).append(value) + super().__init__(tmp) # type: ignore[arg-type] + + def __getstate__(self) -> t.Any: + return dict(self.lists()) + + def __setstate__(self, value: t.Any) -> None: + super().clear() + super().update(value) + + def __iter__(self) -> cabc.Iterator[K]: + # https://github.com/python/cpython/issues/87412 + # If __iter__ is not overridden, Python uses a fast path for dict(md), + # taking the data directly and getting lists of values, rather than + # calling __getitem__ and getting only the first value. + return super().__iter__() + + def __getitem__(self, key: K) -> V: + """Return the first data value for this key; + raises KeyError if not found. + + :param key: The key to be looked up. + :raise KeyError: if the key does not exist. + """ + + if key in self: + lst = super().__getitem__(key) + if len(lst) > 0: # type: ignore[arg-type] + return lst[0] # type: ignore[index,no-any-return] + raise exceptions.BadRequestKeyError(key) + + def __setitem__(self, key: K, value: V) -> None: + """Like :meth:`add` but removes an existing key first. + + :param key: the key for the value. + :param value: the value to set. + """ + super().__setitem__(key, [value]) # type: ignore[assignment] + + def add(self, key: K, value: V) -> None: + """Adds a new value for the key. + + .. versionadded:: 0.6 + + :param key: the key for the value. + :param value: the value to add. + """ + super().setdefault(key, []).append(value) # type: ignore[arg-type,attr-defined] + + @t.overload + def getlist(self, key: K) -> list[V]: ... + @t.overload + def getlist(self, key: K, type: cabc.Callable[[V], T]) -> list[T]: ... + def getlist( + self, key: K, type: cabc.Callable[[V], T] | None = None + ) -> list[V] | list[T]: + """Return the list of items for a given key. If that key is not in the + `MultiDict`, the return value will be an empty list. Just like `get`, + `getlist` accepts a `type` parameter. All items will be converted + with the callable defined there. + + :param key: The key to be looked up. + :param type: Callable to convert each value. If a ``ValueError`` or + ``TypeError`` is raised, the value is omitted. + :return: a :class:`list` of all the values for the key. + + .. versionchanged:: 3.1 + Catches ``TypeError`` in addition to ``ValueError``. + """ + try: + rv: list[V] = super().__getitem__(key) # type: ignore[assignment] + except KeyError: + return [] + if type is None: + return list(rv) + result = [] + for item in rv: + try: + result.append(type(item)) + except (ValueError, TypeError): + pass + return result + + def setlist(self, key: K, new_list: cabc.Iterable[V]) -> None: + """Remove the old values for a key and add new ones. Note that the list + you pass the values in will be shallow-copied before it is inserted in + the dictionary. + + >>> d = MultiDict() + >>> d.setlist('foo', ['1', '2']) + >>> d['foo'] + '1' + >>> d.getlist('foo') + ['1', '2'] + + :param key: The key for which the values are set. + :param new_list: An iterable with the new values for the key. Old values + are removed first. + """ + super().__setitem__(key, list(new_list)) # type: ignore[assignment] + + @t.overload + def setdefault(self, key: K) -> None: ... + @t.overload + def setdefault(self, key: K, default: V) -> V: ... + def setdefault(self, key: K, default: V | None = None) -> V | None: + """Returns the value for the key if it is in the dict, otherwise it + returns `default` and sets that value for `key`. + + :param key: The key to be looked up. + :param default: The default value to be returned if the key is not + in the dict. If not further specified it's `None`. + """ + if key not in self: + self[key] = default # type: ignore[assignment] + + return self[key] + + def setlistdefault( + self, key: K, default_list: cabc.Iterable[V] | None = None + ) -> list[V]: + """Like `setdefault` but sets multiple values. The list returned + is not a copy, but the list that is actually used internally. This + means that you can put new values into the dict by appending items + to the list: + + >>> d = MultiDict({"foo": 1}) + >>> d.setlistdefault("foo").extend([2, 3]) + >>> d.getlist("foo") + [1, 2, 3] + + :param key: The key to be looked up. + :param default_list: An iterable of default values. It is either copied + (in case it was a list) or converted into a list + before returned. + :return: a :class:`list` + """ + if key not in self: + super().__setitem__(key, list(default_list or ())) # type: ignore[assignment] + + return super().__getitem__(key) # type: ignore[return-value] + + def items(self, multi: bool = False) -> cabc.Iterable[tuple[K, V]]: # type: ignore[override] + """Return an iterator of ``(key, value)`` pairs. + + :param multi: If set to `True` the iterator returned will have a pair + for each value of each key. Otherwise it will only + contain pairs for the first value of each key. + """ + values: list[V] + + for key, values in super().items(): # type: ignore[assignment] + if multi: + for value in values: + yield key, value + else: + yield key, values[0] + + def lists(self) -> cabc.Iterable[tuple[K, list[V]]]: + """Return a iterator of ``(key, values)`` pairs, where values is the list + of all values associated with the key.""" + values: list[V] + + for key, values in super().items(): # type: ignore[assignment] + yield key, list(values) + + def values(self) -> cabc.Iterable[V]: # type: ignore[override] + """Returns an iterator of the first value on every key's value list.""" + values: list[V] + + for values in super().values(): # type: ignore[assignment] + yield values[0] + + def listvalues(self) -> cabc.Iterable[list[V]]: + """Return an iterator of all values associated with a key. Zipping + :meth:`keys` and this is the same as calling :meth:`lists`: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> zip(d.keys(), d.listvalues()) == d.lists() + True + """ + return super().values() # type: ignore[return-value] + + def copy(self) -> te.Self: + """Return a shallow copy of this object.""" + return self.__class__(self) + + def deepcopy(self, memo: t.Any = None) -> te.Self: + """Return a deep copy of this object.""" + return self.__class__(deepcopy(self.to_dict(flat=False), memo)) + + @t.overload + def to_dict(self, flat: t.Literal[True] = ...) -> dict[K, V]: ... + @t.overload + def to_dict(self, flat: t.Literal[False]) -> dict[K, list[V]]: ... + def to_dict(self, flat: bool = True) -> dict[K, V] | dict[K, list[V]]: + """Return the contents as regular dict. If `flat` is `True` the + returned dict will only have the first item present, if `flat` is + `False` all values will be returned as lists. + + :param flat: If set to `False` the dict returned will have lists + with all the values in it. Otherwise it will only + contain the first value for each key. + :return: a :class:`dict` + """ + if flat: + return dict(self.items()) + return dict(self.lists()) + + def update( # type: ignore[override] + self, + mapping: ( + MultiDict[K, V] + | cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + | cabc.Iterable[tuple[K, V]] + ), + ) -> None: + """update() extends rather than replaces existing key lists: + + >>> a = MultiDict({'x': 1}) + >>> b = MultiDict({'x': 2, 'y': 3}) + >>> a.update(b) + >>> a + MultiDict([('y', 3), ('x', 1), ('x', 2)]) + + If the value list for a key in ``other_dict`` is empty, no new values + will be added to the dict and the key will not be created: + + >>> x = {'empty_list': []} + >>> y = MultiDict() + >>> y.update(x) + >>> y + MultiDict([]) + """ + for key, value in iter_multi_items(mapping): + self.add(key, value) + + def __or__( # type: ignore[override] + self, other: cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + ) -> MultiDict[K, V]: + if not isinstance(other, cabc.Mapping): + return NotImplemented + + rv = self.copy() + rv.update(other) + return rv + + def __ior__( # type: ignore[override] + self, + other: ( + cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + | cabc.Iterable[tuple[K, V]] + ), + ) -> te.Self: + if not isinstance(other, (cabc.Mapping, cabc.Iterable)): + return NotImplemented + + self.update(other) + return self + + @t.overload + def pop(self, key: K) -> V: ... + @t.overload + def pop(self, key: K, default: V) -> V: ... + @t.overload + def pop(self, key: K, default: T) -> V | T: ... + def pop( + self, + key: K, + default: V | T = _missing, # type: ignore[assignment] + ) -> V | T: + """Pop the first item for a list on the dict. Afterwards the + key is removed from the dict, so additional values are discarded: + + >>> d = MultiDict({"foo": [1, 2, 3]}) + >>> d.pop("foo") + 1 + >>> "foo" in d + False + + :param key: the key to pop. + :param default: if provided the value to return if the key was + not in the dictionary. + """ + lst: list[V] + + try: + lst = super().pop(key) # type: ignore[assignment] + + if len(lst) == 0: + raise exceptions.BadRequestKeyError(key) + + return lst[0] + except KeyError: + if default is not _missing: + return default + + raise exceptions.BadRequestKeyError(key) from None + + def popitem(self) -> tuple[K, V]: + """Pop an item from the dict.""" + item: tuple[K, list[V]] + + try: + item = super().popitem() # type: ignore[assignment] + + if len(item[1]) == 0: + raise exceptions.BadRequestKeyError(item[0]) + + return item[0], item[1][0] + except KeyError as e: + raise exceptions.BadRequestKeyError(e.args[0]) from None + + def poplist(self, key: K) -> list[V]: + """Pop the list for a key from the dict. If the key is not in the dict + an empty list is returned. + + .. versionchanged:: 0.5 + If the key does no longer exist a list is returned instead of + raising an error. + """ + return super().pop(key, []) # type: ignore[return-value] + + def popitemlist(self) -> tuple[K, list[V]]: + """Pop a ``(key, list)`` tuple from the dict.""" + try: + return super().popitem() # type: ignore[return-value] + except KeyError as e: + raise exceptions.BadRequestKeyError(e.args[0]) from None + + def __copy__(self) -> te.Self: + return self.copy() + + def __deepcopy__(self, memo: t.Any) -> te.Self: + return self.deepcopy(memo=memo) + + def __repr__(self) -> str: + return f"{type(self).__name__}({list(self.items(multi=True))!r})" + + +class _omd_bucket(t.Generic[K, V]): + """Wraps values in the :class:`OrderedMultiDict`. This makes it + possible to keep an order over multiple different keys. It requires + a lot of extra memory and slows down access a lot, but makes it + possible to access elements in O(1) and iterate in O(n). + """ + + __slots__ = ("prev", "key", "value", "next") + + def __init__(self, omd: _OrderedMultiDict[K, V], key: K, value: V) -> None: + self.prev: _omd_bucket[K, V] | None = omd._last_bucket + self.key: K = key + self.value: V = value + self.next: _omd_bucket[K, V] | None = None + + if omd._first_bucket is None: + omd._first_bucket = self + if omd._last_bucket is not None: + omd._last_bucket.next = self + omd._last_bucket = self + + def unlink(self, omd: _OrderedMultiDict[K, V]) -> None: + if self.prev: + self.prev.next = self.next + if self.next: + self.next.prev = self.prev + if omd._first_bucket is self: + omd._first_bucket = self.next + if omd._last_bucket is self: + omd._last_bucket = self.prev + + +class _OrderedMultiDict(MultiDict[K, V]): + """Works like a regular :class:`MultiDict` but preserves the + order of the fields. To convert the ordered multi dict into a + list you can use the :meth:`items` method and pass it ``multi=True``. + + In general an :class:`OrderedMultiDict` is an order of magnitude + slower than a :class:`MultiDict`. + + .. admonition:: note + + Due to a limitation in Python you cannot convert an ordered + multi dict into a regular dict by using ``dict(multidict)``. + Instead you have to use the :meth:`to_dict` method, otherwise + the internal bucket objects are exposed. + + .. deprecated:: 3.1 + Will be removed in Werkzeug 3.2. Use ``MultiDict`` instead. + """ + + def __init__( + self, + mapping: ( + MultiDict[K, V] + | cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + | cabc.Iterable[tuple[K, V]] + | None + ) = None, + ) -> None: + import warnings + + warnings.warn( + "'OrderedMultiDict' is deprecated and will be removed in Werkzeug" + " 3.2. Use 'MultiDict' instead.", + DeprecationWarning, + stacklevel=2, + ) + super().__init__() + self._first_bucket: _omd_bucket[K, V] | None = None + self._last_bucket: _omd_bucket[K, V] | None = None + if mapping is not None: + self.update(mapping) + + def __eq__(self, other: object) -> bool: + if not isinstance(other, MultiDict): + return NotImplemented + if isinstance(other, _OrderedMultiDict): + iter1 = iter(self.items(multi=True)) + iter2 = iter(other.items(multi=True)) + try: + for k1, v1 in iter1: + k2, v2 = next(iter2) + if k1 != k2 or v1 != v2: + return False + except StopIteration: + return False + try: + next(iter2) + except StopIteration: + return True + return False + if len(self) != len(other): + return False + for key, values in self.lists(): + if other.getlist(key) != values: + return False + return True + + __hash__ = None # type: ignore[assignment] + + def __reduce_ex__(self, protocol: t.SupportsIndex) -> t.Any: + return type(self), (list(self.items(multi=True)),) + + def __getstate__(self) -> t.Any: + return list(self.items(multi=True)) + + def __setstate__(self, values: t.Any) -> None: + self.clear() + + for key, value in values: + self.add(key, value) + + def __getitem__(self, key: K) -> V: + if key in self: + return dict.__getitem__(self, key)[0].value # type: ignore[index,no-any-return] + raise exceptions.BadRequestKeyError(key) + + def __setitem__(self, key: K, value: V) -> None: + self.poplist(key) + self.add(key, value) + + def __delitem__(self, key: K) -> None: + self.pop(key) + + def keys(self) -> cabc.Iterable[K]: # type: ignore[override] + return (key for key, _ in self.items()) + + def __iter__(self) -> cabc.Iterator[K]: + return iter(self.keys()) + + def values(self) -> cabc.Iterable[V]: # type: ignore[override] + return (value for key, value in self.items()) + + def items(self, multi: bool = False) -> cabc.Iterable[tuple[K, V]]: # type: ignore[override] + ptr = self._first_bucket + if multi: + while ptr is not None: + yield ptr.key, ptr.value + ptr = ptr.next + else: + returned_keys = set() + while ptr is not None: + if ptr.key not in returned_keys: + returned_keys.add(ptr.key) + yield ptr.key, ptr.value + ptr = ptr.next + + def lists(self) -> cabc.Iterable[tuple[K, list[V]]]: + returned_keys = set() + ptr = self._first_bucket + while ptr is not None: + if ptr.key not in returned_keys: + yield ptr.key, self.getlist(ptr.key) + returned_keys.add(ptr.key) + ptr = ptr.next + + def listvalues(self) -> cabc.Iterable[list[V]]: + for _key, values in self.lists(): + yield values + + def add(self, key: K, value: V) -> None: + dict.setdefault(self, key, []).append(_omd_bucket(self, key, value)) # type: ignore[misc] + + @t.overload + def getlist(self, key: K) -> list[V]: ... + @t.overload + def getlist(self, key: K, type: cabc.Callable[[V], T]) -> list[T]: ... + def getlist( + self, key: K, type: cabc.Callable[[V], T] | None = None + ) -> list[V] | list[T]: + rv: list[_omd_bucket[K, V]] + + try: + rv = dict.__getitem__(self, key) # type: ignore[index] + except KeyError: + return [] + if type is None: + return [x.value for x in rv] + result = [] + for item in rv: + try: + result.append(type(item.value)) + except (ValueError, TypeError): + pass + return result + + def setlist(self, key: K, new_list: cabc.Iterable[V]) -> None: + self.poplist(key) + for value in new_list: + self.add(key, value) + + def setlistdefault(self, key: t.Any, default_list: t.Any = None) -> t.NoReturn: + raise TypeError("setlistdefault is unsupported for ordered multi dicts") + + def update( # type: ignore[override] + self, + mapping: ( + MultiDict[K, V] + | cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + | cabc.Iterable[tuple[K, V]] + ), + ) -> None: + for key, value in iter_multi_items(mapping): + self.add(key, value) + + def poplist(self, key: K) -> list[V]: + buckets: cabc.Iterable[_omd_bucket[K, V]] = dict.pop(self, key, ()) # type: ignore[arg-type] + for bucket in buckets: + bucket.unlink(self) + return [x.value for x in buckets] + + @t.overload + def pop(self, key: K) -> V: ... + @t.overload + def pop(self, key: K, default: V) -> V: ... + @t.overload + def pop(self, key: K, default: T) -> V | T: ... + def pop( + self, + key: K, + default: V | T = _missing, # type: ignore[assignment] + ) -> V | T: + buckets: list[_omd_bucket[K, V]] + + try: + buckets = dict.pop(self, key) # type: ignore[arg-type] + except KeyError: + if default is not _missing: + return default + + raise exceptions.BadRequestKeyError(key) from None + + for bucket in buckets: + bucket.unlink(self) + + return buckets[0].value + + def popitem(self) -> tuple[K, V]: + key: K + buckets: list[_omd_bucket[K, V]] + + try: + key, buckets = dict.popitem(self) # type: ignore[arg-type] + except KeyError as e: + raise exceptions.BadRequestKeyError(e.args[0]) from None + + for bucket in buckets: + bucket.unlink(self) + + return key, buckets[0].value + + def popitemlist(self) -> tuple[K, list[V]]: + key: K + buckets: list[_omd_bucket[K, V]] + + try: + key, buckets = dict.popitem(self) # type: ignore[arg-type] + except KeyError as e: + raise exceptions.BadRequestKeyError(e.args[0]) from None + + for bucket in buckets: + bucket.unlink(self) + + return key, [x.value for x in buckets] + + +class CombinedMultiDict(ImmutableMultiDictMixin[K, V], MultiDict[K, V]): # type: ignore[misc] + """A read only :class:`MultiDict` that you can pass multiple :class:`MultiDict` + instances as sequence and it will combine the return values of all wrapped + dicts: + + >>> from werkzeug.datastructures import CombinedMultiDict, MultiDict + >>> post = MultiDict([('foo', 'bar')]) + >>> get = MultiDict([('blub', 'blah')]) + >>> combined = CombinedMultiDict([get, post]) + >>> combined['foo'] + 'bar' + >>> combined['blub'] + 'blah' + + This works for all read operations and will raise a `TypeError` for + methods that usually change data which isn't possible. + + From Werkzeug 0.3 onwards, the `KeyError` raised by this class is also a + subclass of the :exc:`~exceptions.BadRequest` HTTP exception and will + render a page for a ``400 BAD REQUEST`` if caught in a catch-all for HTTP + exceptions. + """ + + def __reduce_ex__(self, protocol: t.SupportsIndex) -> t.Any: + return type(self), (self.dicts,) + + def __init__(self, dicts: cabc.Iterable[MultiDict[K, V]] | None = None) -> None: + super().__init__() + self.dicts: list[MultiDict[K, V]] = list(dicts or ()) + + @classmethod + def fromkeys(cls, keys: t.Any, value: t.Any = None) -> t.NoReturn: + raise TypeError(f"cannot create {cls.__name__!r} instances by fromkeys") + + def __getitem__(self, key: K) -> V: + for d in self.dicts: + if key in d: + return d[key] + raise exceptions.BadRequestKeyError(key) + + @t.overload # type: ignore[override] + def get(self, key: K) -> V | None: ... + @t.overload + def get(self, key: K, default: V) -> V: ... + @t.overload + def get(self, key: K, default: T) -> V | T: ... + @t.overload + def get(self, key: str, type: cabc.Callable[[V], T]) -> T | None: ... + @t.overload + def get(self, key: str, default: T, type: cabc.Callable[[V], T]) -> T: ... + def get( # type: ignore[misc] + self, + key: K, + default: V | T | None = None, + type: cabc.Callable[[V], T] | None = None, + ) -> V | T | None: + for d in self.dicts: + if key in d: + if type is not None: + try: + return type(d[key]) + except (ValueError, TypeError): + continue + return d[key] + return default + + @t.overload + def getlist(self, key: K) -> list[V]: ... + @t.overload + def getlist(self, key: K, type: cabc.Callable[[V], T]) -> list[T]: ... + def getlist( + self, key: K, type: cabc.Callable[[V], T] | None = None + ) -> list[V] | list[T]: + rv = [] + for d in self.dicts: + rv.extend(d.getlist(key, type)) # type: ignore[arg-type] + return rv + + def _keys_impl(self) -> set[K]: + """This function exists so __len__ can be implemented more efficiently, + saving one list creation from an iterator. + """ + return set(k for d in self.dicts for k in d) + + def keys(self) -> cabc.Iterable[K]: # type: ignore[override] + return self._keys_impl() + + def __iter__(self) -> cabc.Iterator[K]: + return iter(self._keys_impl()) + + @t.overload # type: ignore[override] + def items(self) -> cabc.Iterable[tuple[K, V]]: ... + @t.overload + def items(self, multi: t.Literal[True]) -> cabc.Iterable[tuple[K, list[V]]]: ... + def items( + self, multi: bool = False + ) -> cabc.Iterable[tuple[K, V]] | cabc.Iterable[tuple[K, list[V]]]: + found = set() + for d in self.dicts: + for key, value in d.items(multi): + if multi: + yield key, value + elif key not in found: + found.add(key) + yield key, value + + def values(self) -> cabc.Iterable[V]: # type: ignore[override] + for _, value in self.items(): + yield value + + def lists(self) -> cabc.Iterable[tuple[K, list[V]]]: + rv: dict[K, list[V]] = {} + for d in self.dicts: + for key, values in d.lists(): + rv.setdefault(key, []).extend(values) + return rv.items() + + def listvalues(self) -> cabc.Iterable[list[V]]: + return (x[1] for x in self.lists()) + + def copy(self) -> MultiDict[K, V]: # type: ignore[override] + """Return a shallow mutable copy of this object. + + This returns a :class:`MultiDict` representing the data at the + time of copying. The copy will no longer reflect changes to the + wrapped dicts. + + .. versionchanged:: 0.15 + Return a mutable :class:`MultiDict`. + """ + return MultiDict(self) + + def __len__(self) -> int: + return len(self._keys_impl()) + + def __contains__(self, key: K) -> bool: # type: ignore[override] + for d in self.dicts: + if key in d: + return True + return False + + def __repr__(self) -> str: + return f"{type(self).__name__}({self.dicts!r})" + + +class ImmutableDict(ImmutableDictMixin[K, V], dict[K, V]): # type: ignore[misc] + """An immutable :class:`dict`. + + .. versionadded:: 0.5 + """ + + def __repr__(self) -> str: + return f"{type(self).__name__}({dict.__repr__(self)})" + + def copy(self) -> dict[K, V]: + """Return a shallow mutable copy of this object. Keep in mind that + the standard library's :func:`copy` function is a no-op for this class + like for any other python immutable type (eg: :class:`tuple`). + """ + return dict(self) + + def __copy__(self) -> te.Self: + return self + + +class ImmutableMultiDict(ImmutableMultiDictMixin[K, V], MultiDict[K, V]): # type: ignore[misc] + """An immutable :class:`MultiDict`. + + .. versionadded:: 0.5 + """ + + def copy(self) -> MultiDict[K, V]: # type: ignore[override] + """Return a shallow mutable copy of this object. Keep in mind that + the standard library's :func:`copy` function is a no-op for this class + like for any other python immutable type (eg: :class:`tuple`). + """ + return MultiDict(self) + + def __copy__(self) -> te.Self: + return self + + +class _ImmutableOrderedMultiDict( # type: ignore[misc] + ImmutableMultiDictMixin[K, V], _OrderedMultiDict[K, V] +): + """An immutable :class:`OrderedMultiDict`. + + .. deprecated:: 3.1 + Will be removed in Werkzeug 3.2. Use ``ImmutableMultiDict`` instead. + + .. versionadded:: 0.6 + """ + + def __init__( + self, + mapping: ( + MultiDict[K, V] + | cabc.Mapping[K, V | list[V] | tuple[V, ...] | set[V]] + | cabc.Iterable[tuple[K, V]] + | None + ) = None, + ) -> None: + super().__init__() + + if mapping is not None: + for k, v in iter_multi_items(mapping): + _OrderedMultiDict.add(self, k, v) + + def _iter_hashitems(self) -> cabc.Iterable[t.Any]: + return enumerate(self.items(multi=True)) + + def copy(self) -> _OrderedMultiDict[K, V]: # type: ignore[override] + """Return a shallow mutable copy of this object. Keep in mind that + the standard library's :func:`copy` function is a no-op for this class + like for any other python immutable type (eg: :class:`tuple`). + """ + return _OrderedMultiDict(self) + + def __copy__(self) -> te.Self: + return self + + +class CallbackDict(UpdateDictMixin[K, V], dict[K, V]): + """A dict that calls a function passed every time something is changed. + The function is passed the dict instance. + """ + + def __init__( + self, + initial: cabc.Mapping[K, V] | cabc.Iterable[tuple[K, V]] | None = None, + on_update: cabc.Callable[[te.Self], None] | None = None, + ) -> None: + if initial is None: + super().__init__() + else: + super().__init__(initial) + + self.on_update = on_update + + def __repr__(self) -> str: + return f"<{type(self).__name__} {super().__repr__()}>" + + +class HeaderSet(cabc.MutableSet[str]): + """Similar to the :class:`ETags` class this implements a set-like structure. + Unlike :class:`ETags` this is case insensitive and used for vary, allow, and + content-language headers. + + If not constructed using the :func:`parse_set_header` function the + instantiation works like this: + + >>> hs = HeaderSet(['foo', 'bar', 'baz']) + >>> hs + HeaderSet(['foo', 'bar', 'baz']) + """ + + def __init__( + self, + headers: cabc.Iterable[str] | None = None, + on_update: cabc.Callable[[te.Self], None] | None = None, + ) -> None: + self._headers = list(headers or ()) + self._set = {x.lower() for x in self._headers} + self.on_update = on_update + + def add(self, header: str) -> None: + """Add a new header to the set.""" + self.update((header,)) + + def remove(self: te.Self, header: str) -> None: + """Remove a header from the set. This raises an :exc:`KeyError` if the + header is not in the set. + + .. versionchanged:: 0.5 + In older versions a :exc:`IndexError` was raised instead of a + :exc:`KeyError` if the object was missing. + + :param header: the header to be removed. + """ + key = header.lower() + if key not in self._set: + raise KeyError(header) + self._set.remove(key) + for idx, key in enumerate(self._headers): + if key.lower() == header: + del self._headers[idx] + break + if self.on_update is not None: + self.on_update(self) + + def update(self: te.Self, iterable: cabc.Iterable[str]) -> None: + """Add all the headers from the iterable to the set. + + :param iterable: updates the set with the items from the iterable. + """ + inserted_any = False + for header in iterable: + key = header.lower() + if key not in self._set: + self._headers.append(header) + self._set.add(key) + inserted_any = True + if inserted_any and self.on_update is not None: + self.on_update(self) + + def discard(self, header: str) -> None: + """Like :meth:`remove` but ignores errors. + + :param header: the header to be discarded. + """ + try: + self.remove(header) + except KeyError: + pass + + def find(self, header: str) -> int: + """Return the index of the header in the set or return -1 if not found. + + :param header: the header to be looked up. + """ + header = header.lower() + for idx, item in enumerate(self._headers): + if item.lower() == header: + return idx + return -1 + + def index(self, header: str) -> int: + """Return the index of the header in the set or raise an + :exc:`IndexError`. + + :param header: the header to be looked up. + """ + rv = self.find(header) + if rv < 0: + raise IndexError(header) + return rv + + def clear(self: te.Self) -> None: + """Clear the set.""" + self._set.clear() + self._headers.clear() + + if self.on_update is not None: + self.on_update(self) + + def as_set(self, preserve_casing: bool = False) -> set[str]: + """Return the set as real python set type. When calling this, all + the items are converted to lowercase and the ordering is lost. + + :param preserve_casing: if set to `True` the items in the set returned + will have the original case like in the + :class:`HeaderSet`, otherwise they will + be lowercase. + """ + if preserve_casing: + return set(self._headers) + return set(self._set) + + def to_header(self) -> str: + """Convert the header set into an HTTP header string.""" + return ", ".join(map(http.quote_header_value, self._headers)) + + def __getitem__(self, idx: t.SupportsIndex) -> str: + return self._headers[idx] + + def __delitem__(self: te.Self, idx: t.SupportsIndex) -> None: + rv = self._headers.pop(idx) + self._set.remove(rv.lower()) + if self.on_update is not None: + self.on_update(self) + + def __setitem__(self: te.Self, idx: t.SupportsIndex, value: str) -> None: + old = self._headers[idx] + self._set.remove(old.lower()) + self._headers[idx] = value + self._set.add(value.lower()) + if self.on_update is not None: + self.on_update(self) + + def __contains__(self, header: str) -> bool: # type: ignore[override] + return header.lower() in self._set + + def __len__(self) -> int: + return len(self._set) + + def __iter__(self) -> cabc.Iterator[str]: + return iter(self._headers) + + def __bool__(self) -> bool: + return bool(self._set) + + def __str__(self) -> str: + return self.to_header() + + def __repr__(self) -> str: + return f"{type(self).__name__}({self._headers!r})" + + +# circular dependencies +from .. import http # noqa: E402 + + +def __getattr__(name: str) -> t.Any: + import warnings + + if name == "OrderedMultiDict": + warnings.warn( + "'OrderedMultiDict' is deprecated and will be removed in Werkzeug" + " 3.2. Use 'MultiDict' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _OrderedMultiDict + + if name == "ImmutableOrderedMultiDict": + warnings.warn( + "'ImmutableOrderedMultiDict' is deprecated and will be removed in" + " Werkzeug 3.2. Use 'ImmutableMultiDict' instead.", + DeprecationWarning, + stacklevel=2, + ) + return _ImmutableOrderedMultiDict + + raise AttributeError(name) diff --git a/lib/python3.12/site-packages/werkzeug/debug/__init__.py b/lib/python3.12/site-packages/werkzeug/debug/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..17cd4b25a9d2826329f7f2711f45eee4aebd9c82 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/debug/__init__.py @@ -0,0 +1,574 @@ +from __future__ import annotations + +import getpass +import hashlib +import json +import os +import pkgutil +import re +import sys +import time +import typing as t +import uuid +from contextlib import ExitStack +from io import BytesIO +from itertools import chain +from multiprocessing import Value +from os.path import basename +from os.path import join +from zlib import adler32 + +from .._internal import _log +from ..exceptions import NotFound +from ..exceptions import SecurityError +from ..http import parse_cookie +from ..sansio.utils import host_is_trusted +from ..security import gen_salt +from ..utils import send_file +from ..wrappers.request import Request +from ..wrappers.response import Response +from .console import Console +from .tbtools import DebugFrameSummary +from .tbtools import DebugTraceback +from .tbtools import render_console_html + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + +# A week +PIN_TIME = 60 * 60 * 24 * 7 + + +def hash_pin(pin: str) -> str: + return hashlib.sha1(f"{pin} added salt".encode("utf-8", "replace")).hexdigest()[:12] + + +_machine_id: str | bytes | None = None + + +def get_machine_id() -> str | bytes | None: + global _machine_id + + if _machine_id is not None: + return _machine_id + + def _generate() -> str | bytes | None: + linux = b"" + + # machine-id is stable across boots, boot_id is not. + for filename in "/etc/machine-id", "/proc/sys/kernel/random/boot_id": + try: + with open(filename, "rb") as f: + value = f.readline().strip() + except OSError: + continue + + if value: + linux += value + break + + # Containers share the same machine id, add some cgroup + # information. This is used outside containers too but should be + # relatively stable across boots. + try: + with open("/proc/self/cgroup", "rb") as f: + linux += f.readline().strip().rpartition(b"/")[2] + except OSError: + pass + + if linux: + return linux + + # On OS X, use ioreg to get the computer's serial number. + try: + # subprocess may not be available, e.g. Google App Engine + # https://github.com/pallets/werkzeug/issues/925 + from subprocess import PIPE + from subprocess import Popen + + dump = Popen( + ["ioreg", "-c", "IOPlatformExpertDevice", "-d", "2"], stdout=PIPE + ).communicate()[0] + match = re.search(b'"serial-number" = <([^>]+)', dump) + + if match is not None: + return match.group(1) + except (OSError, ImportError): + pass + + # On Windows, use winreg to get the machine guid. + if sys.platform == "win32": + import winreg + + try: + with winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + "SOFTWARE\\Microsoft\\Cryptography", + 0, + winreg.KEY_READ | winreg.KEY_WOW64_64KEY, + ) as rk: + guid: str | bytes + guid_type: int + guid, guid_type = winreg.QueryValueEx(rk, "MachineGuid") + + if guid_type == winreg.REG_SZ: + return guid.encode() + + return guid + except OSError: + pass + + return None + + _machine_id = _generate() + return _machine_id + + +class _ConsoleFrame: + """Helper class so that we can reuse the frame console code for the + standalone console. + """ + + def __init__(self, namespace: dict[str, t.Any]): + self.console = Console(namespace) + self.id = 0 + + def eval(self, code: str) -> t.Any: + return self.console.eval(code) + + +def get_pin_and_cookie_name( + app: WSGIApplication, +) -> tuple[str, str] | tuple[None, None]: + """Given an application object this returns a semi-stable 9 digit pin + code and a random key. The hope is that this is stable between + restarts to not make debugging particularly frustrating. If the pin + was forcefully disabled this returns `None`. + + Second item in the resulting tuple is the cookie name for remembering. + """ + pin = os.environ.get("WERKZEUG_DEBUG_PIN") + rv = None + num = None + + # Pin was explicitly disabled + if pin == "off": + return None, None + + # Pin was provided explicitly + if pin is not None and pin.replace("-", "").isdecimal(): + # If there are separators in the pin, return it directly + if "-" in pin: + rv = pin + else: + num = pin + + modname = getattr(app, "__module__", t.cast(object, app).__class__.__module__) + username: str | None + + try: + # getuser imports the pwd module, which does not exist in Google + # App Engine. It may also raise a KeyError if the UID does not + # have a username, such as in Docker. + username = getpass.getuser() + # Python >= 3.13 only raises OSError + except (ImportError, KeyError, OSError): + username = None + + mod = sys.modules.get(modname) + + # This information only exists to make the cookie unique on the + # computer, not as a security feature. + probably_public_bits = [ + username, + modname, + getattr(app, "__name__", type(app).__name__), + getattr(mod, "__file__", None), + ] + + # This information is here to make it harder for an attacker to + # guess the cookie name. They are unlikely to be contained anywhere + # within the unauthenticated debug page. + private_bits = [str(uuid.getnode()), get_machine_id()] + + h = hashlib.sha1() + for bit in chain(probably_public_bits, private_bits): + if not bit: + continue + if isinstance(bit, str): + bit = bit.encode() + h.update(bit) + h.update(b"cookiesalt") + + cookie_name = f"__wzd{h.hexdigest()[:20]}" + + # If we need to generate a pin we salt it a bit more so that we don't + # end up with the same value and generate out 9 digits + if num is None: + h.update(b"pinsalt") + num = f"{int(h.hexdigest(), 16):09d}"[:9] + + # Format the pincode in groups of digits for easier remembering if + # we don't have a result yet. + if rv is None: + for group_size in 5, 4, 3: + if len(num) % group_size == 0: + rv = "-".join( + num[x : x + group_size].rjust(group_size, "0") + for x in range(0, len(num), group_size) + ) + break + else: + rv = num + + return rv, cookie_name + + +class DebuggedApplication: + """Enables debugging support for a given application:: + + from werkzeug.debug import DebuggedApplication + from myapp import app + app = DebuggedApplication(app, evalex=True) + + The ``evalex`` argument allows evaluating expressions in any frame + of a traceback. This works by preserving each frame with its local + state. Some state, such as context globals, cannot be restored with + the frame by default. When ``evalex`` is enabled, + ``environ["werkzeug.debug.preserve_context"]`` will be a callable + that takes a context manager, and can be called multiple times. + Each context manager will be entered before evaluating code in the + frame, then exited again, so they can perform setup and cleanup for + each call. + + :param app: the WSGI application to run debugged. + :param evalex: enable exception evaluation feature (interactive + debugging). This requires a non-forking server. + :param request_key: The key that points to the request object in this + environment. This parameter is ignored in current + versions. + :param console_path: the URL for a general purpose console. + :param console_init_func: the function that is executed before starting + the general purpose console. The return value + is used as initial namespace. + :param show_hidden_frames: by default hidden traceback frames are skipped. + You can show them by setting this parameter + to `True`. + :param pin_security: can be used to disable the pin based security system. + :param pin_logging: enables the logging of the pin system. + + .. versionchanged:: 2.2 + Added the ``werkzeug.debug.preserve_context`` environ key. + """ + + _pin: str | None + _pin_cookie: str + + def __init__( + self, + app: WSGIApplication, + evalex: bool = False, + request_key: str = "werkzeug.request", + console_path: str = "/console", + console_init_func: t.Callable[[], dict[str, t.Any]] | None = None, + show_hidden_frames: bool = False, + pin_security: bool = True, + pin_logging: bool = True, + ) -> None: + if not console_init_func: + console_init_func = None + self.app = app + self.evalex = evalex + self.frames: dict[int, DebugFrameSummary | _ConsoleFrame] = {} + self.frame_contexts: dict[int, list[t.ContextManager[None]]] = {} + self.request_key = request_key + self.console_path = console_path + self.console_init_func = console_init_func + self.show_hidden_frames = show_hidden_frames + self.secret = gen_salt(20) + self._failed_pin_auth = Value("B") + + self.pin_logging = pin_logging + if pin_security: + # Print out the pin for the debugger on standard out. + if os.environ.get("WERKZEUG_RUN_MAIN") == "true" and pin_logging: + _log("warning", " * Debugger is active!") + if self.pin is None: + _log("warning", " * Debugger PIN disabled. DEBUGGER UNSECURED!") + else: + _log("info", " * Debugger PIN: %s", self.pin) + else: + self.pin = None + + self.trusted_hosts: list[str] = [".localhost", "127.0.0.1"] + """List of domains to allow requests to the debugger from. A leading dot + allows all subdomains. This only allows ``".localhost"`` domains by + default. + + .. versionadded:: 3.0.3 + """ + + @property + def pin(self) -> str | None: + if not hasattr(self, "_pin"): + pin_cookie = get_pin_and_cookie_name(self.app) + self._pin, self._pin_cookie = pin_cookie # type: ignore + return self._pin + + @pin.setter + def pin(self, value: str | None) -> None: + if value is None: + # Set _pin to None explicitly to prevent regeneration by the getter + self._pin = None + else: + self._pin = value + + @property + def pin_cookie_name(self) -> str: + """The name of the pin cookie.""" + if not hasattr(self, "_pin_cookie"): + pin_cookie = get_pin_and_cookie_name(self.app) + self._pin, self._pin_cookie = pin_cookie # type: ignore + return self._pin_cookie + + def debug_application( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterator[bytes]: + """Run the application and conserve the traceback frames.""" + contexts: list[t.ContextManager[t.Any]] = [] + + if self.evalex: + environ["werkzeug.debug.preserve_context"] = contexts.append + + app_iter = None + try: + app_iter = self.app(environ, start_response) + yield from app_iter + if hasattr(app_iter, "close"): + app_iter.close() + except Exception as e: + if hasattr(app_iter, "close"): + app_iter.close() # type: ignore + + tb = DebugTraceback(e, skip=1, hide=not self.show_hidden_frames) + + for frame in tb.all_frames: + self.frames[id(frame)] = frame + self.frame_contexts[id(frame)] = contexts + + is_trusted = bool(self.check_pin_trust(environ)) + html = tb.render_debugger_html( + evalex=self.evalex and self.check_host_trust(environ), + secret=self.secret, + evalex_trusted=is_trusted, + ) + response = Response(html, status=500, mimetype="text/html") + + try: + yield from response(environ, start_response) + except Exception: + # if we end up here there has been output but an error + # occurred. in that situation we can do nothing fancy any + # more, better log something into the error log and fall + # back gracefully. + environ["wsgi.errors"].write( + "Debugging middleware caught exception in streamed " + "response at a point where response headers were already " + "sent.\n" + ) + + environ["wsgi.errors"].write("".join(tb.render_traceback_text())) + + def execute_command( + self, + request: Request, + command: str, + frame: DebugFrameSummary | _ConsoleFrame, + ) -> Response: + """Execute a command in a console.""" + if not self.check_host_trust(request.environ): + return SecurityError() # type: ignore[return-value] + + contexts = self.frame_contexts.get(id(frame), []) + + with ExitStack() as exit_stack: + for cm in contexts: + exit_stack.enter_context(cm) + + return Response(frame.eval(command), mimetype="text/html") + + def display_console(self, request: Request) -> Response: + """Display a standalone shell.""" + if not self.check_host_trust(request.environ): + return SecurityError() # type: ignore[return-value] + + if 0 not in self.frames: + if self.console_init_func is None: + ns = {} + else: + ns = dict(self.console_init_func()) + ns.setdefault("app", self.app) + self.frames[0] = _ConsoleFrame(ns) + is_trusted = bool(self.check_pin_trust(request.environ)) + return Response( + render_console_html(secret=self.secret, evalex_trusted=is_trusted), + mimetype="text/html", + ) + + def get_resource(self, request: Request, filename: str) -> Response: + """Return a static resource from the shared folder.""" + path = join("shared", basename(filename)) + + try: + data = pkgutil.get_data(__package__, path) + except OSError: + return NotFound() # type: ignore[return-value] + else: + if data is None: + return NotFound() # type: ignore[return-value] + + etag = str(adler32(data) & 0xFFFFFFFF) + return send_file( + BytesIO(data), request.environ, download_name=filename, etag=etag + ) + + def check_pin_trust(self, environ: WSGIEnvironment) -> bool | None: + """Checks if the request passed the pin test. This returns `True` if the + request is trusted on a pin/cookie basis and returns `False` if not. + Additionally if the cookie's stored pin hash is wrong it will return + `None` so that appropriate action can be taken. + """ + if self.pin is None: + return True + + # If we failed too many times, then we're locked out. + if self._failed_pin_auth.value >= 10: + return False + + val = parse_cookie(environ).get(self.pin_cookie_name) + if not val or "|" not in val: + return False + ts_str, pin_hash = val.split("|", 1) + + try: + ts = int(ts_str) + except ValueError: + return False + + if pin_hash != hash_pin(self.pin): + return None + return (time.time() - PIN_TIME) < ts + + def check_host_trust(self, environ: WSGIEnvironment) -> bool: + return host_is_trusted(environ.get("HTTP_HOST"), self.trusted_hosts) + + def _fail_pin_auth(self) -> None: + with self._failed_pin_auth.get_lock(): + count = self._failed_pin_auth.value + self._failed_pin_auth.value = count + 1 + + time.sleep(5.0 if count > 5 else 0.5) + + def pin_auth(self, request: Request) -> Response: + """Authenticates with the pin.""" + if not self.check_host_trust(request.environ): + return SecurityError() # type: ignore[return-value] + + exhausted = False + auth = False + trust = self.check_pin_trust(request.environ) + pin = t.cast(str, self.pin) + + # If the trust return value is `None` it means that the cookie is + # set but the stored pin hash value is bad. This means that the + # pin was changed. In this case we count a bad auth and unset the + # cookie. This way it becomes harder to guess the cookie name + # instead of the pin as we still count up failures. + bad_cookie = False + if trust is None: + self._fail_pin_auth() + bad_cookie = True + + # If we're trusted, we're authenticated. + elif trust: + auth = True + + # If we failed too many times, then we're locked out. + elif self._failed_pin_auth.value >= 10: + exhausted = True + + # Otherwise go through pin based authentication + else: + entered_pin = request.args["pin"] + + if entered_pin.strip().replace("-", "") == pin.replace("-", ""): + self._failed_pin_auth.value = 0 + auth = True + else: + self._fail_pin_auth() + + rv = Response( + json.dumps({"auth": auth, "exhausted": exhausted}), + mimetype="application/json", + ) + if auth: + rv.set_cookie( + self.pin_cookie_name, + f"{int(time.time())}|{hash_pin(pin)}", + httponly=True, + samesite="Strict", + secure=request.is_secure, + ) + elif bad_cookie: + rv.delete_cookie(self.pin_cookie_name) + return rv + + def log_pin_request(self, request: Request) -> Response: + """Log the pin if needed.""" + if not self.check_host_trust(request.environ): + return SecurityError() # type: ignore[return-value] + + if self.pin_logging and self.pin is not None: + _log( + "info", " * To enable the debugger you need to enter the security pin:" + ) + _log("info", " * Debugger pin code: %s", self.pin) + return Response("") + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + """Dispatch the requests.""" + # important: don't ever access a function here that reads the incoming + # form data! Otherwise the application won't have access to that data + # any more! + request = Request(environ) + response = self.debug_application + if request.args.get("__debugger__") == "yes": + cmd = request.args.get("cmd") + arg = request.args.get("f") + secret = request.args.get("s") + frame = self.frames.get(request.args.get("frm", type=int)) # type: ignore + if cmd == "resource" and arg: + response = self.get_resource(request, arg) # type: ignore + elif cmd == "pinauth" and secret == self.secret: + response = self.pin_auth(request) # type: ignore + elif cmd == "printpin" and secret == self.secret: + response = self.log_pin_request(request) # type: ignore + elif ( + self.evalex + and cmd is not None + and frame is not None + and self.secret == secret + and self.check_pin_trust(environ) + ): + response = self.execute_command(request, cmd, frame) # type: ignore + elif ( + self.evalex + and self.console_path is not None + and request.path == self.console_path + ): + response = self.display_console(request) # type: ignore + return response(environ, start_response) diff --git a/lib/python3.12/site-packages/werkzeug/debug/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ee7bc45e235c57db9dbfeabb07d44a88f5dd9429 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/debug/__pycache__/console.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/console.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f01a7ade094d642746aee22b3b5826ee784b6d4 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/console.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/debug/__pycache__/repr.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/repr.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f72e848fa9b0169bee9540b666469eaecfce10e7 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/repr.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/debug/__pycache__/tbtools.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/tbtools.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5e20cf103aec27ef88fa6e4428966ded80284533 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/debug/__pycache__/tbtools.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/debug/console.py b/lib/python3.12/site-packages/werkzeug/debug/console.py new file mode 100644 index 0000000000000000000000000000000000000000..4e40475a5351ddccc0ef72746a159a6a75da63c2 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/debug/console.py @@ -0,0 +1,219 @@ +from __future__ import annotations + +import code +import sys +import typing as t +from contextvars import ContextVar +from types import CodeType + +from markupsafe import escape + +from .repr import debug_repr +from .repr import dump +from .repr import helper + +_stream: ContextVar[HTMLStringO] = ContextVar("werkzeug.debug.console.stream") +_ipy: ContextVar[_InteractiveConsole] = ContextVar("werkzeug.debug.console.ipy") + + +class HTMLStringO: + """A StringO version that HTML escapes on write.""" + + def __init__(self) -> None: + self._buffer: list[str] = [] + + def isatty(self) -> bool: + return False + + def close(self) -> None: + pass + + def flush(self) -> None: + pass + + def seek(self, n: int, mode: int = 0) -> None: + pass + + def readline(self) -> str: + if len(self._buffer) == 0: + return "" + ret = self._buffer[0] + del self._buffer[0] + return ret + + def reset(self) -> str: + val = "".join(self._buffer) + del self._buffer[:] + return val + + def _write(self, x: str) -> None: + self._buffer.append(x) + + def write(self, x: str) -> None: + self._write(escape(x)) + + def writelines(self, x: t.Iterable[str]) -> None: + self._write(escape("".join(x))) + + +class ThreadedStream: + """Thread-local wrapper for sys.stdout for the interactive console.""" + + @staticmethod + def push() -> None: + if not isinstance(sys.stdout, ThreadedStream): + sys.stdout = t.cast(t.TextIO, ThreadedStream()) + + _stream.set(HTMLStringO()) + + @staticmethod + def fetch() -> str: + try: + stream = _stream.get() + except LookupError: + return "" + + return stream.reset() + + @staticmethod + def displayhook(obj: object) -> None: + try: + stream = _stream.get() + except LookupError: + return _displayhook(obj) # type: ignore + + # stream._write bypasses escaping as debug_repr is + # already generating HTML for us. + if obj is not None: + _ipy.get().locals["_"] = obj + stream._write(debug_repr(obj)) + + def __setattr__(self, name: str, value: t.Any) -> None: + raise AttributeError(f"read only attribute {name}") + + def __dir__(self) -> list[str]: + return dir(sys.__stdout__) + + def __getattribute__(self, name: str) -> t.Any: + try: + stream = _stream.get() + except LookupError: + stream = sys.__stdout__ # type: ignore[assignment] + + return getattr(stream, name) + + def __repr__(self) -> str: + return repr(sys.__stdout__) + + +# add the threaded stream as display hook +_displayhook = sys.displayhook +sys.displayhook = ThreadedStream.displayhook + + +class _ConsoleLoader: + def __init__(self) -> None: + self._storage: dict[int, str] = {} + + def register(self, code: CodeType, source: str) -> None: + self._storage[id(code)] = source + # register code objects of wrapped functions too. + for var in code.co_consts: + if isinstance(var, CodeType): + self._storage[id(var)] = source + + def get_source_by_code(self, code: CodeType) -> str | None: + try: + return self._storage[id(code)] + except KeyError: + return None + + +class _InteractiveConsole(code.InteractiveInterpreter): + locals: dict[str, t.Any] + + def __init__(self, globals: dict[str, t.Any], locals: dict[str, t.Any]) -> None: + self.loader = _ConsoleLoader() + locals = { + **globals, + **locals, + "dump": dump, + "help": helper, + "__loader__": self.loader, + } + super().__init__(locals) + original_compile = self.compile + + def compile(source: str, filename: str, symbol: str) -> CodeType | None: + code = original_compile(source, filename, symbol) + + if code is not None: + self.loader.register(code, source) + + return code + + self.compile = compile # type: ignore[assignment] + self.more = False + self.buffer: list[str] = [] + + def runsource(self, source: str, **kwargs: t.Any) -> str: # type: ignore + source = f"{source.rstrip()}\n" + ThreadedStream.push() + prompt = "... " if self.more else ">>> " + try: + source_to_eval = "".join(self.buffer + [source]) + if super().runsource(source_to_eval, "", "single"): + self.more = True + self.buffer.append(source) + else: + self.more = False + del self.buffer[:] + finally: + output = ThreadedStream.fetch() + return f"{prompt}{escape(source)}{output}" + + def runcode(self, code: CodeType) -> None: + try: + exec(code, self.locals) + except Exception: + self.showtraceback() + + def showtraceback(self) -> None: + from .tbtools import DebugTraceback + + exc = t.cast(BaseException, sys.exc_info()[1]) + te = DebugTraceback(exc, skip=1) + sys.stdout._write(te.render_traceback_html()) # type: ignore + + def showsyntaxerror(self, filename: str | None = None) -> None: + from .tbtools import DebugTraceback + + exc = t.cast(BaseException, sys.exc_info()[1]) + te = DebugTraceback(exc, skip=4) + sys.stdout._write(te.render_traceback_html()) # type: ignore + + def write(self, data: str) -> None: + sys.stdout.write(data) + + +class Console: + """An interactive console.""" + + def __init__( + self, + globals: dict[str, t.Any] | None = None, + locals: dict[str, t.Any] | None = None, + ) -> None: + if locals is None: + locals = {} + if globals is None: + globals = {} + self._ipy = _InteractiveConsole(globals, locals) + + def eval(self, code: str) -> str: + _ipy.set(self._ipy) + old_sys_stdout = sys.stdout + try: + return self._ipy.runsource(code) + finally: + sys.stdout = old_sys_stdout diff --git a/lib/python3.12/site-packages/werkzeug/debug/repr.py b/lib/python3.12/site-packages/werkzeug/debug/repr.py new file mode 100644 index 0000000000000000000000000000000000000000..2bbd9d5467f1c7adea105b038ab817ad6171ddad --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/debug/repr.py @@ -0,0 +1,282 @@ +"""Object representations for debugging purposes. Unlike the default +repr, these expose more information and produce HTML instead of ASCII. + +Together with the CSS and JavaScript of the debugger this gives a +colorful and more compact output. +""" + +from __future__ import annotations + +import codecs +import re +import sys +import typing as t +from collections import deque +from traceback import format_exception_only + +from markupsafe import escape + +missing = object() +_paragraph_re = re.compile(r"(?:\r\n|\r|\n){2,}") +RegexType = type(_paragraph_re) + +HELP_HTML = """\ +
+

%(title)s

+ 
%(text)s
+
\ +""" +OBJECT_DUMP_HTML = """\ +
+

%(title)s

+ %(repr)s + %(items)s
+
\ +""" + + +def debug_repr(obj: object) -> str: + """Creates a debug repr of an object as HTML string.""" + return DebugReprGenerator().repr(obj) + + +def dump(obj: object = missing) -> None: + """Print the object details to stdout._write (for the interactive + console of the web debugger. + """ + gen = DebugReprGenerator() + if obj is missing: + rv = gen.dump_locals(sys._getframe(1).f_locals) + else: + rv = gen.dump_object(obj) + sys.stdout._write(rv) # type: ignore + + +class _Helper: + """Displays an HTML version of the normal help, for the interactive + debugger only because it requires a patched sys.stdout. + """ + + def __repr__(self) -> str: + return "Type help(object) for help about object." + + def __call__(self, topic: t.Any | None = None) -> None: + if topic is None: + sys.stdout._write(f"{self!r}") # type: ignore + return + import pydoc + + pydoc.help(topic) + rv = sys.stdout.reset() # type: ignore + paragraphs = _paragraph_re.split(rv) + if len(paragraphs) > 1: + title = paragraphs[0] + text = "\n\n".join(paragraphs[1:]) + else: + title = "Help" + text = paragraphs[0] + sys.stdout._write(HELP_HTML % {"title": title, "text": text}) # type: ignore + + +helper = _Helper() + + +def _add_subclass_info(inner: str, obj: object, base: type | tuple[type, ...]) -> str: + if isinstance(base, tuple): + for cls in base: + if type(obj) is cls: + return inner + elif type(obj) is base: + return inner + module = "" + if obj.__class__.__module__ not in ("__builtin__", "exceptions"): + module = f'{obj.__class__.__module__}.' + return f"{module}{type(obj).__name__}({inner})" + + +def _sequence_repr_maker( + left: str, right: str, base: type, limit: int = 8 +) -> t.Callable[[DebugReprGenerator, t.Iterable[t.Any], bool], str]: + def proxy(self: DebugReprGenerator, obj: t.Iterable[t.Any], recursive: bool) -> str: + if recursive: + return _add_subclass_info(f"{left}...{right}", obj, base) + buf = [left] + have_extended_section = False + for idx, item in enumerate(obj): + if idx: + buf.append(", ") + if idx == limit: + buf.append('') + have_extended_section = True + buf.append(self.repr(item)) + if have_extended_section: + buf.append("") + buf.append(right) + return _add_subclass_info("".join(buf), obj, base) + + return proxy + + +class DebugReprGenerator: + def __init__(self) -> None: + self._stack: list[t.Any] = [] + + list_repr = _sequence_repr_maker("[", "]", list) + tuple_repr = _sequence_repr_maker("(", ")", tuple) + set_repr = _sequence_repr_maker("set([", "])", set) + frozenset_repr = _sequence_repr_maker("frozenset([", "])", frozenset) + deque_repr = _sequence_repr_maker( + 'collections.deque([', "])", deque + ) + + def regex_repr(self, obj: t.Pattern[t.AnyStr]) -> str: + pattern = repr(obj.pattern) + pattern = codecs.decode(pattern, "unicode-escape", "ignore") + pattern = f"r{pattern}" + return f're.compile({pattern})' + + def string_repr(self, obj: str | bytes, limit: int = 70) -> str: + buf = [''] + r = repr(obj) + + # shorten the repr when the hidden part would be at least 3 chars + if len(r) - limit > 2: + buf.extend( + ( + escape(r[:limit]), + '', + escape(r[limit:]), + "", + ) + ) + else: + buf.append(escape(r)) + + buf.append("") + out = "".join(buf) + + # if the repr looks like a standard string, add subclass info if needed + if r[0] in "'\"" or (r[0] == "b" and r[1] in "'\""): + return _add_subclass_info(out, obj, (bytes, str)) + + # otherwise, assume the repr distinguishes the subclass already + return out + + def dict_repr( + self, + d: dict[int, None] | dict[str, int] | dict[str | int, int], + recursive: bool, + limit: int = 5, + ) -> str: + if recursive: + return _add_subclass_info("{...}", d, dict) + buf = ["{"] + have_extended_section = False + for idx, (key, value) in enumerate(d.items()): + if idx: + buf.append(", ") + if idx == limit - 1: + buf.append('') + have_extended_section = True + buf.append( + f'{self.repr(key)}:' + f' {self.repr(value)}' + ) + if have_extended_section: + buf.append("") + buf.append("}") + return _add_subclass_info("".join(buf), d, dict) + + def object_repr(self, obj: t.Any) -> str: + r = repr(obj) + return f'{escape(r)}' + + def dispatch_repr(self, obj: t.Any, recursive: bool) -> str: + if obj is helper: + return f'{helper!r}' + if isinstance(obj, (int, float, complex)): + return f'{obj!r}' + if isinstance(obj, str) or isinstance(obj, bytes): + return self.string_repr(obj) + if isinstance(obj, RegexType): + return self.regex_repr(obj) + if isinstance(obj, list): + return self.list_repr(obj, recursive) + if isinstance(obj, tuple): + return self.tuple_repr(obj, recursive) + if isinstance(obj, set): + return self.set_repr(obj, recursive) + if isinstance(obj, frozenset): + return self.frozenset_repr(obj, recursive) + if isinstance(obj, dict): + return self.dict_repr(obj, recursive) + if isinstance(obj, deque): + return self.deque_repr(obj, recursive) + return self.object_repr(obj) + + def fallback_repr(self) -> str: + try: + info = "".join(format_exception_only(*sys.exc_info()[:2])) + except Exception: + info = "?" + return ( + '' + f"<broken repr ({escape(info.strip())})>" + ) + + def repr(self, obj: object) -> str: + recursive = False + for item in self._stack: + if item is obj: + recursive = True + break + self._stack.append(obj) + try: + try: + return self.dispatch_repr(obj, recursive) + except Exception: + return self.fallback_repr() + finally: + self._stack.pop() + + def dump_object(self, obj: object) -> str: + repr = None + items: list[tuple[str, str]] | None = None + + if isinstance(obj, dict): + title = "Contents of" + items = [] + for key, value in obj.items(): + if not isinstance(key, str): + items = None + break + items.append((key, self.repr(value))) + if items is None: + items = [] + repr = self.repr(obj) + for key in dir(obj): + try: + items.append((key, self.repr(getattr(obj, key)))) + except Exception: + pass + title = "Details for" + title += f" {object.__repr__(obj)[1:-1]}" + return self.render_object_dump(items, title, repr) + + def dump_locals(self, d: dict[str, t.Any]) -> str: + items = [(key, self.repr(value)) for key, value in d.items()] + return self.render_object_dump(items, "Local variables in frame") + + def render_object_dump( + self, items: list[tuple[str, str]], title: str, repr: str | None = None + ) -> str: + html_items = [] + for key, value in items: + html_items.append(f"{escape(key)}
{value}
") + if not html_items: + html_items.append("Nothing") + return OBJECT_DUMP_HTML % { + "title": escape(title), + "repr": f"
{repr if repr else ''}
", + "items": "\n".join(html_items), + } diff --git a/lib/python3.12/site-packages/werkzeug/debug/shared/ICON_LICENSE.md b/lib/python3.12/site-packages/werkzeug/debug/shared/ICON_LICENSE.md new file mode 100644 index 0000000000000000000000000000000000000000..3bdbfc739ea9c4d80d02f1a495cb0ee2f7be9b97 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/debug/shared/ICON_LICENSE.md @@ -0,0 +1,6 @@ +Silk icon set 1.3 by Mark James + +http://www.famfamfam.com/lab/icons/silk/ + +License: [CC-BY-2.5](https://creativecommons.org/licenses/by/2.5/) +or [CC-BY-3.0](https://creativecommons.org/licenses/by/3.0/) diff --git a/lib/python3.12/site-packages/werkzeug/debug/shared/console.png b/lib/python3.12/site-packages/werkzeug/debug/shared/console.png new file mode 100644 index 0000000000000000000000000000000000000000..c28dd63812d80e416682f835652f8e5824bdccb2 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/debug/shared/console.png differ diff --git a/lib/python3.12/site-packages/werkzeug/debug/shared/debugger.js b/lib/python3.12/site-packages/werkzeug/debug/shared/debugger.js new file mode 100644 index 0000000000000000000000000000000000000000..809b14a6edaff688a619a7203e0619b9b4d101ff --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/debug/shared/debugger.js @@ -0,0 +1,344 @@ +docReady(() => { + if (!EVALEX_TRUSTED) { + initPinBox(); + } + // if we are in console mode, show the console. + if (CONSOLE_MODE && EVALEX) { + createInteractiveConsole(); + } + + const frames = document.querySelectorAll("div.traceback div.frame"); + if (EVALEX) { + addConsoleIconToFrames(frames); + } + addEventListenersToElements(document.querySelectorAll("div.detail"), "click", () => + document.querySelector("div.traceback").scrollIntoView(false) + ); + addToggleFrameTraceback(frames); + addToggleTraceTypesOnClick(document.querySelectorAll("h2.traceback")); + addInfoPrompt(document.querySelectorAll("span.nojavascript")); + wrapPlainTraceback(); +}); + +function addToggleFrameTraceback(frames) { + frames.forEach((frame) => { + frame.addEventListener("click", () => { + frame.getElementsByTagName("pre")[0].parentElement.classList.toggle("expanded"); + }); + }) +} + + +function wrapPlainTraceback() { + const plainTraceback = document.querySelector("div.plain textarea"); + const wrapper = document.createElement("pre"); + const textNode = document.createTextNode(plainTraceback.textContent); + wrapper.appendChild(textNode); + plainTraceback.replaceWith(wrapper); +} + +function makeDebugURL(args) { + const params = new URLSearchParams(args) + params.set("s", SECRET) + return `?__debugger__=yes&${params}` +} + +function initPinBox() { + document.querySelector(".pin-prompt form").addEventListener( + "submit", + function (event) { + event.preventDefault(); + const btn = this.btn; + btn.disabled = true; + + fetch( + makeDebugURL({cmd: "pinauth", pin: this.pin.value}) + ) + .then((res) => res.json()) + .then(({auth, exhausted}) => { + if (auth) { + EVALEX_TRUSTED = true; + fadeOut(document.getElementsByClassName("pin-prompt")[0]); + } else { + alert( + `Error: ${ + exhausted + ? "too many attempts. Restart server to retry." + : "incorrect pin" + }` + ); + } + }) + .catch((err) => { + alert("Error: Could not verify PIN. Network error?"); + console.error(err); + }) + .finally(() => (btn.disabled = false)); + }, + false + ); +} + +function promptForPin() { + if (!EVALEX_TRUSTED) { + fetch(makeDebugURL({cmd: "printpin"})); + const pinPrompt = document.getElementsByClassName("pin-prompt")[0]; + fadeIn(pinPrompt); + document.querySelector('.pin-prompt input[name="pin"]').focus(); + } +} + +/** + * Helper function for shell initialization + */ +function openShell(consoleNode, target, frameID) { + promptForPin(); + if (consoleNode) { + slideToggle(consoleNode); + return consoleNode; + } + let historyPos = 0; + const history = [""]; + const consoleElement = createConsole(); + const output = createConsoleOutput(); + const form = createConsoleInputForm(); + const command = createConsoleInput(); + + target.parentNode.appendChild(consoleElement); + consoleElement.append(output); + consoleElement.append(form); + form.append(command); + command.focus(); + slideToggle(consoleElement); + + form.addEventListener("submit", (e) => { + handleConsoleSubmit(e, command, frameID).then((consoleOutput) => { + output.append(consoleOutput); + command.focus(); + consoleElement.scrollTo(0, consoleElement.scrollHeight); + const old = history.pop(); + history.push(command.value); + if (typeof old !== "undefined") { + history.push(old); + } + historyPos = history.length - 1; + command.value = ""; + }); + }); + + command.addEventListener("keydown", (e) => { + if (e.key === "l" && e.ctrlKey) { + output.innerText = "--- screen cleared ---"; + } else if (e.key === "ArrowUp" || e.key === "ArrowDown") { + // Handle up arrow and down arrow. + if (e.key === "ArrowUp" && historyPos > 0) { + e.preventDefault(); + historyPos--; + } else if (e.key === "ArrowDown" && historyPos < history.length - 1) { + historyPos++; + } + command.value = history[historyPos]; + } + return false; + }); + + return consoleElement; +} + +function addEventListenersToElements(elements, event, listener) { + elements.forEach((el) => el.addEventListener(event, listener)); +} + +/** + * Add extra info + */ +function addInfoPrompt(elements) { + for (let i = 0; i < elements.length; i++) { + elements[i].innerHTML = + "

To switch between the interactive traceback and the plaintext " + + 'one, you can click on the "Traceback" headline. From the text ' + + "traceback you can also create a paste of it. " + + (!EVALEX + ? "" + : "For code execution mouse-over the frame you want to debug and " + + "click on the console icon on the right side." + + "

You can execute arbitrary Python code in the stack frames and " + + "there are some extra helpers available for introspection:" + + "

  • dump() shows all variables in the frame" + + "
  • dump(obj) dumps all that's known about the object
"); + elements[i].classList.remove("nojavascript"); + } +} + +function addConsoleIconToFrames(frames) { + for (let i = 0; i < frames.length; i++) { + let consoleNode = null; + const target = frames[i]; + const frameID = frames[i].id.substring(6); + + for (let j = 0; j < target.getElementsByTagName("pre").length; j++) { + const img = createIconForConsole(); + img.addEventListener("click", (e) => { + e.stopPropagation(); + consoleNode = openShell(consoleNode, target, frameID); + return false; + }); + target.getElementsByTagName("pre")[j].append(img); + } + } +} + +function slideToggle(target) { + target.classList.toggle("active"); +} + +/** + * toggle traceback types on click. + */ +function addToggleTraceTypesOnClick(elements) { + for (let i = 0; i < elements.length; i++) { + elements[i].addEventListener("click", () => { + document.querySelector("div.traceback").classList.toggle("hidden"); + document.querySelector("div.plain").classList.toggle("hidden"); + }); + elements[i].style.cursor = "pointer"; + document.querySelector("div.plain").classList.toggle("hidden"); + } +} + +function createConsole() { + const consoleNode = document.createElement("pre"); + consoleNode.classList.add("console"); + consoleNode.classList.add("active"); + return consoleNode; +} + +function createConsoleOutput() { + const output = document.createElement("div"); + output.classList.add("output"); + output.innerHTML = "[console ready]"; + return output; +} + +function createConsoleInputForm() { + const form = document.createElement("form"); + form.innerHTML = ">>> "; + return form; +} + +function createConsoleInput() { + const command = document.createElement("input"); + command.type = "text"; + command.setAttribute("autocomplete", "off"); + command.setAttribute("spellcheck", false); + command.setAttribute("autocapitalize", "off"); + command.setAttribute("autocorrect", "off"); + return command; +} + +function createIconForConsole() { + const img = document.createElement("img"); + img.setAttribute("src", makeDebugURL({cmd: "resource", f: "console.png"})); + img.setAttribute("title", "Open an interactive python shell in this frame"); + return img; +} + +function createExpansionButtonForConsole() { + const expansionButton = document.createElement("a"); + expansionButton.setAttribute("href", "#"); + expansionButton.setAttribute("class", "toggle"); + expansionButton.innerHTML = "  "; + return expansionButton; +} + +function createInteractiveConsole() { + const target = document.querySelector("div.console div.inner"); + while (target.firstChild) { + target.removeChild(target.firstChild); + } + openShell(null, target, 0); +} + +function handleConsoleSubmit(e, command, frameID) { + // Prevent page from refreshing. + e.preventDefault(); + + return new Promise((resolve) => { + fetch(makeDebugURL({cmd: command.value, frm: frameID})) + .then((res) => { + return res.text(); + }) + .then((data) => { + const tmp = document.createElement("div"); + tmp.innerHTML = data; + resolve(tmp); + + // Handle expandable span for long list outputs. + // Example to test: list(range(13)) + let wrapperAdded = false; + const wrapperSpan = document.createElement("span"); + const expansionButton = createExpansionButtonForConsole(); + + tmp.querySelectorAll("span.extended").forEach((spanToWrap) => { + const parentDiv = spanToWrap.parentNode; + if (!wrapperAdded) { + parentDiv.insertBefore(wrapperSpan, spanToWrap); + wrapperAdded = true; + } + parentDiv.removeChild(spanToWrap); + wrapperSpan.append(spanToWrap); + spanToWrap.hidden = true; + + expansionButton.addEventListener("click", (event) => { + event.preventDefault(); + spanToWrap.hidden = !spanToWrap.hidden; + expansionButton.classList.toggle("open"); + return false; + }); + }); + + // Add expansion button at end of wrapper. + if (wrapperAdded) { + wrapperSpan.append(expansionButton); + } + }) + .catch((err) => { + console.error(err); + }); + return false; + }); +} + +function fadeOut(element) { + element.style.opacity = 1; + + (function fade() { + element.style.opacity -= 0.1; + if (element.style.opacity < 0) { + element.style.display = "none"; + } else { + requestAnimationFrame(fade); + } + })(); +} + +function fadeIn(element, display) { + element.style.opacity = 0; + element.style.display = display || "block"; + + (function fade() { + let val = parseFloat(element.style.opacity) + 0.1; + if (val <= 1) { + element.style.opacity = val; + requestAnimationFrame(fade); + } + })(); +} + +function docReady(fn) { + if (document.readyState === "complete" || document.readyState === "interactive") { + setTimeout(fn, 1); + } else { + document.addEventListener("DOMContentLoaded", fn); + } +} diff --git a/lib/python3.12/site-packages/werkzeug/debug/shared/less.png b/lib/python3.12/site-packages/werkzeug/debug/shared/less.png new file mode 100644 index 0000000000000000000000000000000000000000..5efefd62b43e4f11dd300be4355a4b413c7a70d2 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/debug/shared/less.png differ diff --git a/lib/python3.12/site-packages/werkzeug/debug/shared/more.png b/lib/python3.12/site-packages/werkzeug/debug/shared/more.png new file mode 100644 index 0000000000000000000000000000000000000000..804fa226fe3ed9e6cc2bd044a848f33a2d7b4e4f Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/debug/shared/more.png differ diff --git a/lib/python3.12/site-packages/werkzeug/debug/shared/style.css b/lib/python3.12/site-packages/werkzeug/debug/shared/style.css new file mode 100644 index 0000000000000000000000000000000000000000..e9397ca0a1b6c26f30cb28fc81510a48fc46ede9 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/debug/shared/style.css @@ -0,0 +1,150 @@ +body, input { font-family: sans-serif; color: #000; text-align: center; + margin: 1em; padding: 0; font-size: 15px; } +h1, h2, h3 { font-weight: normal; } + +input { background-color: #fff; margin: 0; text-align: left; + outline: none !important; } +input[type="submit"] { padding: 3px 6px; } +a { color: #11557C; } +a:hover { color: #177199; } +pre, code, +textarea { font-family: monospace; font-size: 14px; } + +div.debugger { text-align: left; padding: 12px; margin: auto; + background-color: white; } +h1 { font-size: 36px; margin: 0 0 0.3em 0; } +div.detail { cursor: pointer; } +div.detail p { margin: 0 0 8px 13px; font-size: 14px; white-space: pre-wrap; + font-family: monospace; } +div.explanation { margin: 20px 13px; font-size: 15px; color: #555; } +div.footer { font-size: 13px; text-align: right; margin: 30px 0; + color: #86989B; } + +h2 { font-size: 16px; margin: 1.3em 0 0.0 0; padding: 9px; + background-color: #11557C; color: white; } +h2 em, h3 em { font-style: normal; color: #A5D6D9; font-weight: normal; } + +div.traceback, div.plain { border: 1px solid #ddd; margin: 0 0 1em 0; padding: 10px; } +div.plain p { margin: 0; } +div.plain textarea, +div.plain pre { margin: 10px 0 0 0; padding: 4px; + background-color: #E8EFF0; border: 1px solid #D3E7E9; } +div.plain textarea { width: 99%; height: 300px; } +div.traceback h3 { font-size: 1em; margin: 0 0 0.8em 0; } +div.traceback ul { list-style: none; margin: 0; padding: 0 0 0 1em; } +div.traceback h4 { font-size: 13px; font-weight: normal; margin: 0.7em 0 0.1em 0; } +div.traceback pre { margin: 0; padding: 5px 0 3px 15px; + background-color: #E8EFF0; border: 1px solid #D3E7E9; } +div.traceback .library .current { background: white; color: #555; } +div.traceback .expanded .current { background: #E8EFF0; color: black; } +div.traceback pre:hover { background-color: #DDECEE; color: black; cursor: pointer; } +div.traceback div.source.expanded pre + pre { border-top: none; } + +div.traceback span.ws { display: none; } +div.traceback pre.before, div.traceback pre.after { display: none; background: white; } +div.traceback div.source.expanded pre.before, +div.traceback div.source.expanded pre.after { + display: block; +} + +div.traceback div.source.expanded span.ws { + display: inline; +} + +div.traceback blockquote { margin: 1em 0 0 0; padding: 0; white-space: pre-line; } +div.traceback img { float: right; padding: 2px; margin: -3px 2px 0 0; display: none; } +div.traceback img:hover { background-color: #ddd; cursor: pointer; + border-color: #BFDDE0; } +div.traceback pre:hover img { display: block; } +div.traceback cite.filename { font-style: normal; color: #3B666B; } + +pre.console { border: 1px solid #ccc; background: white!important; + color: black; padding: 5px!important; + margin: 3px 0 0 0!important; cursor: default!important; + max-height: 400px; overflow: auto; } +pre.console form { color: #555; } +pre.console input { background-color: transparent; color: #555; + width: 90%; font-family: monospace; font-size: 14px; + border: none!important; } + +span.string { color: #30799B; } +span.number { color: #9C1A1C; } +span.help { color: #3A7734; } +span.object { color: #485F6E; } +span.extended { opacity: 0.5; } +span.extended:hover { opacity: 1; } +a.toggle { text-decoration: none; background-repeat: no-repeat; + background-position: center center; + background-image: url(?__debugger__=yes&cmd=resource&f=more.png); } +a.toggle:hover { background-color: #444; } +a.open { background-image: url(?__debugger__=yes&cmd=resource&f=less.png); } + +pre.console div.traceback, +pre.console div.box { margin: 5px 10px; white-space: normal; + border: 1px solid #11557C; padding: 10px; + font-family: sans-serif; } +pre.console div.box h3, +pre.console div.traceback h3 { margin: -10px -10px 10px -10px; padding: 5px; + background: #11557C; color: white; } + +pre.console div.traceback pre:hover { cursor: default; background: #E8EFF0; } +pre.console div.traceback pre.syntaxerror { background: inherit; border: none; + margin: 20px -10px -10px -10px; + padding: 10px; border-top: 1px solid #BFDDE0; + background: #E8EFF0; } +pre.console div.noframe-traceback pre.syntaxerror { margin-top: -10px; border: none; } + +pre.console div.box pre.repr { padding: 0; margin: 0; background-color: white; border: none; } +pre.console div.box table { margin-top: 6px; } +pre.console div.box pre { border: none; } +pre.console div.box pre.help { background-color: white; } +pre.console div.box pre.help:hover { cursor: default; } +pre.console table tr { vertical-align: top; } +div.console { border: 1px solid #ccc; padding: 4px; background-color: #fafafa; } + +div.traceback pre, div.console pre { + white-space: pre-wrap; /* css-3 should we be so lucky... */ + white-space: -moz-pre-wrap; /* Mozilla, since 1999 */ + white-space: -pre-wrap; /* Opera 4-6 ?? */ + white-space: -o-pre-wrap; /* Opera 7 ?? */ + word-wrap: break-word; /* Internet Explorer 5.5+ */ + _white-space: pre; /* IE only hack to re-specify in + addition to word-wrap */ +} + + +div.pin-prompt { + position: absolute; + display: none; + top: 0; + bottom: 0; + left: 0; + right: 0; + background: rgba(255, 255, 255, 0.8); +} + +div.pin-prompt .inner { + background: #eee; + padding: 10px 50px; + width: 350px; + margin: 10% auto 0 auto; + border: 1px solid #ccc; + border-radius: 2px; +} + +div.exc-divider { + margin: 0.7em 0 0 -1em; + padding: 0.5em; + background: #11557C; + color: #ddd; + border: 1px solid #ddd; +} + +.console.active { + max-height: 0!important; + display: none; +} + +.hidden { + display: none; +} diff --git a/lib/python3.12/site-packages/werkzeug/debug/tbtools.py b/lib/python3.12/site-packages/werkzeug/debug/tbtools.py new file mode 100644 index 0000000000000000000000000000000000000000..ca4acf60a83f5179faf89643304515deadae9ee3 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/debug/tbtools.py @@ -0,0 +1,449 @@ +from __future__ import annotations + +import itertools +import linecache +import os +import re +import sys +import sysconfig +import traceback +import typing as t + +from markupsafe import escape + +from ..utils import cached_property +from .console import Console + +HEADER = """\ + + + + %(title)s // Werkzeug Debugger + + + + + + +
+""" + +FOOTER = """\ +
+ Brought to you by DON'T PANIC, your + friendly Werkzeug powered traceback interpreter. +
+
+ +
+
+

Console Locked

+

+ The console is locked and needs to be unlocked by entering the PIN. + You can find the PIN printed out on the standard output of your + shell that runs the server. +

+

PIN: + + +

+
+
+ + +""" + +PAGE_HTML = ( + HEADER + + """\ +

%(exception_type)s

+
+

%(exception)s

+
+

Traceback (most recent call last)

+%(summary)s +
+

+ This is the Copy/Paste friendly version of the traceback. +

+ +
+
+ The debugger caught an exception in your WSGI application. You can now + look at the traceback which led to the error. + If you enable JavaScript you can also use additional features such as code + execution (if the evalex feature is enabled), automatic pasting of the + exceptions and much more. +
+""" + + FOOTER + + """ + +""" +) + +CONSOLE_HTML = ( + HEADER + + """\ +

Interactive Console

+
+In this console you can execute Python expressions in the context of the +application. The initial namespace was created by the debugger automatically. +
+
The Console requires JavaScript.
+""" + + FOOTER +) + +SUMMARY_HTML = """\ +
+ %(title)s +
    %(frames)s
+ %(description)s +
+""" + +FRAME_HTML = """\ +
+

File "%(filename)s", + line %(lineno)s, + in %(function_name)s

+
%(lines)s
+
+""" + + +def _process_traceback( + exc: BaseException, + te: traceback.TracebackException | None = None, + *, + skip: int = 0, + hide: bool = True, +) -> traceback.TracebackException: + if te is None: + te = traceback.TracebackException.from_exception(exc, lookup_lines=False) + + # Get the frames the same way StackSummary.extract did, in order + # to match each frame with the FrameSummary to augment. + frame_gen = traceback.walk_tb(exc.__traceback__) + limit = getattr(sys, "tracebacklimit", None) + + if limit is not None: + if limit < 0: + limit = 0 + + frame_gen = itertools.islice(frame_gen, limit) + + if skip: + frame_gen = itertools.islice(frame_gen, skip, None) + del te.stack[:skip] + + new_stack: list[DebugFrameSummary] = [] + hidden = False + + # Match each frame with the FrameSummary that was generated. + # Hide frames using Paste's __traceback_hide__ rules. Replace + # all visible FrameSummary with DebugFrameSummary. + for (f, _), fs in zip(frame_gen, te.stack): + if hide: + hide_value = f.f_locals.get("__traceback_hide__", False) + + if hide_value in {"before", "before_and_this"}: + new_stack = [] + hidden = False + + if hide_value == "before_and_this": + continue + elif hide_value in {"reset", "reset_and_this"}: + hidden = False + + if hide_value == "reset_and_this": + continue + elif hide_value in {"after", "after_and_this"}: + hidden = True + + if hide_value == "after_and_this": + continue + elif hide_value or hidden: + continue + + frame_args: dict[str, t.Any] = { + "filename": fs.filename, + "lineno": fs.lineno, + "name": fs.name, + "locals": f.f_locals, + "globals": f.f_globals, + } + + if sys.version_info >= (3, 11): + frame_args["colno"] = fs.colno + frame_args["end_colno"] = fs.end_colno + + new_stack.append(DebugFrameSummary(**frame_args)) + + # The codeop module is used to compile code from the interactive + # debugger. Hide any codeop frames from the bottom of the traceback. + while new_stack: + module = new_stack[0].global_ns.get("__name__") + + if module is None: + module = new_stack[0].local_ns.get("__name__") + + if module == "codeop": + del new_stack[0] + else: + break + + te.stack[:] = new_stack + + if te.__context__: + context_exc = t.cast(BaseException, exc.__context__) + te.__context__ = _process_traceback(context_exc, te.__context__, hide=hide) + + if te.__cause__: + cause_exc = t.cast(BaseException, exc.__cause__) + te.__cause__ = _process_traceback(cause_exc, te.__cause__, hide=hide) + + return te + + +class DebugTraceback: + __slots__ = ("_te", "_cache_all_tracebacks", "_cache_all_frames") + + def __init__( + self, + exc: BaseException, + te: traceback.TracebackException | None = None, + *, + skip: int = 0, + hide: bool = True, + ) -> None: + self._te = _process_traceback(exc, te, skip=skip, hide=hide) + + def __str__(self) -> str: + return f"<{type(self).__name__} {self._te}>" + + @cached_property + def all_tracebacks( + self, + ) -> list[tuple[str | None, traceback.TracebackException]]: + out: list[tuple[str | None, traceback.TracebackException]] = [] + current: traceback.TracebackException | None = self._te + + while current is not None: + if current.__cause__ is not None: + chained_msg = ( + "The above exception was the direct cause of the" + " following exception" + ) + chained_exc = current.__cause__ + elif current.__context__ is not None and not current.__suppress_context__: + chained_msg = ( + "During handling of the above exception, another exception occurred" + ) + chained_exc = current.__context__ + else: + chained_msg = None + chained_exc = None + + out.append((chained_msg, current)) + current = chained_exc + + return out + + @cached_property + def all_frames(self) -> list[DebugFrameSummary]: + return [ + f # type: ignore[misc] + for _, te in self.all_tracebacks + for f in te.stack + ] + + def render_traceback_text(self) -> str: + return "".join(self._te.format()) + + def render_traceback_html(self, include_title: bool = True) -> str: + library_frames = [f.is_library for f in self.all_frames] + mark_library = 0 < sum(library_frames) < len(library_frames) + rows = [] + + if not library_frames: + classes = "traceback noframe-traceback" + else: + classes = "traceback" + + for msg, current in reversed(self.all_tracebacks): + row_parts = [] + + if msg is not None: + row_parts.append(f'
  • {msg}:
    ') + + for frame in current.stack: + frame = t.cast(DebugFrameSummary, frame) + info = f' title="{escape(frame.info)}"' if frame.info else "" + row_parts.append(f"{frame.render_html(mark_library)}") + + rows.append("\n".join(row_parts)) + + if sys.version_info < (3, 13): + exc_type_str = self._te.exc_type.__name__ + else: + exc_type_str = self._te.exc_type_str + + is_syntax_error = exc_type_str == "SyntaxError" + + if include_title: + if is_syntax_error: + title = "Syntax Error" + else: + title = "Traceback (most recent call last):" + else: + title = "" + + exc_full = escape("".join(self._te.format_exception_only())) + + if is_syntax_error: + description = f"
    {exc_full}
    " + else: + description = f"
    {exc_full}
    " + + return SUMMARY_HTML % { + "classes": classes, + "title": f"

    {title}

    ", + "frames": "\n".join(rows), + "description": description, + } + + def render_debugger_html( + self, evalex: bool, secret: str, evalex_trusted: bool + ) -> str: + exc_lines = list(self._te.format_exception_only()) + plaintext = "".join(self._te.format()) + + if sys.version_info < (3, 13): + exc_type_str = self._te.exc_type.__name__ + else: + exc_type_str = self._te.exc_type_str + + return PAGE_HTML % { + "evalex": "true" if evalex else "false", + "evalex_trusted": "true" if evalex_trusted else "false", + "console": "false", + "title": escape(exc_lines[0]), + "exception": escape("".join(exc_lines)), + "exception_type": escape(exc_type_str), + "summary": self.render_traceback_html(include_title=False), + "plaintext": escape(plaintext), + "plaintext_cs": re.sub("-{2,}", "-", plaintext), + "secret": secret, + } + + +class DebugFrameSummary(traceback.FrameSummary): + """A :class:`traceback.FrameSummary` that can evaluate code in the + frame's namespace. + """ + + __slots__ = ( + "local_ns", + "global_ns", + "_cache_info", + "_cache_is_library", + "_cache_console", + ) + + def __init__( + self, + *, + locals: dict[str, t.Any], + globals: dict[str, t.Any], + **kwargs: t.Any, + ) -> None: + super().__init__(locals=None, **kwargs) + self.local_ns = locals + self.global_ns = globals + + @cached_property + def info(self) -> str | None: + return self.local_ns.get("__traceback_info__") + + @cached_property + def is_library(self) -> bool: + return any( + self.filename.startswith((path, os.path.realpath(path))) + for path in sysconfig.get_paths().values() + ) + + @cached_property + def console(self) -> Console: + return Console(self.global_ns, self.local_ns) + + def eval(self, code: str) -> t.Any: + return self.console.eval(code) + + def render_html(self, mark_library: bool) -> str: + context = 5 + lines = linecache.getlines(self.filename) + line_idx = self.lineno - 1 # type: ignore[operator] + start_idx = max(0, line_idx - context) + stop_idx = min(len(lines), line_idx + context + 1) + rendered_lines = [] + + def render_line(line: str, cls: str) -> None: + line = line.expandtabs().rstrip() + stripped_line = line.strip() + prefix = len(line) - len(stripped_line) + colno = getattr(self, "colno", 0) + end_colno = getattr(self, "end_colno", 0) + + if cls == "current" and colno and end_colno: + arrow = ( + f'\n{" " * prefix}' + f"{' ' * (colno - prefix)}{'^' * (end_colno - colno)}" + ) + else: + arrow = "" + + rendered_lines.append( + f'
    {" " * prefix}'
+                f"{escape(stripped_line) if stripped_line else ' '}"
+                f"{arrow if arrow else ''}
    " + ) + + if line_idx < len(lines): + for line in lines[start_idx:line_idx]: + render_line(line, "before") + + render_line(lines[line_idx], "current") + + for line in lines[line_idx + 1 : stop_idx]: + render_line(line, "after") + + return FRAME_HTML % { + "id": id(self), + "filename": escape(self.filename), + "lineno": self.lineno, + "function_name": escape(self.name), + "lines": "\n".join(rendered_lines), + "library": "library" if mark_library and self.is_library else "", + } + + +def render_console_html(secret: str, evalex_trusted: bool) -> str: + return CONSOLE_HTML % { + "evalex": "true", + "evalex_trusted": "true" if evalex_trusted else "false", + "console": "true", + "title": "Console", + "secret": secret, + } diff --git a/lib/python3.12/site-packages/werkzeug/exceptions.py b/lib/python3.12/site-packages/werkzeug/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..15f534e967993aa4fa8a966abc65e998f837fa91 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/exceptions.py @@ -0,0 +1,905 @@ +"""Implements a number of Python exceptions which can be raised from within +a view to trigger a standard HTTP non-200 response. + +Usage Example +------------- + +.. code-block:: python + + from werkzeug.wrappers.request import Request + from werkzeug.exceptions import HTTPException, NotFound + + def view(request): + raise NotFound() + + @Request.application + def application(request): + try: + return view(request) + except HTTPException as e: + return e + +As you can see from this example those exceptions are callable WSGI +applications. However, they are not Werkzeug response objects. You +can get a response object by calling ``get_response()`` on a HTTP +exception. + +Keep in mind that you may have to pass an environ (WSGI) or scope +(ASGI) to ``get_response()`` because some errors fetch additional +information relating to the request. + +If you want to hook in a different exception page to say, a 404 status +code, you can add a second except for a specific subclass of an error: + +.. code-block:: python + + @Request.application + def application(request): + try: + return view(request) + except NotFound as e: + return not_found(request) + except HTTPException as e: + return e + +""" + +from __future__ import annotations + +import typing as t +from datetime import datetime + +from markupsafe import escape +from markupsafe import Markup + +from ._internal import _get_environ + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIEnvironment + + from .datastructures import WWWAuthenticate + from .sansio.response import Response as SansIOResponse + from .wrappers.request import Request as WSGIRequest + from .wrappers.response import Response as WSGIResponse + + +class HTTPException(Exception): + """The base class for all HTTP exceptions. This exception can be called as a WSGI + application to render a default error page or you can catch the subclasses + of it independently and render nicer error messages. + + .. versionchanged:: 2.1 + Removed the ``wrap`` class method. + """ + + code: int | None = None + description: str | None = None + + def __init__( + self, + description: str | None = None, + response: SansIOResponse | None = None, + ) -> None: + super().__init__() + if description is not None: + self.description = description + self.response = response + + @property + def name(self) -> str: + """The status name.""" + from .http import HTTP_STATUS_CODES + + return HTTP_STATUS_CODES.get(self.code, "Unknown Error") # type: ignore + + def get_description( + self, + environ: WSGIEnvironment | None = None, + scope: dict[str, t.Any] | None = None, + ) -> str: + """Get the description.""" + if self.description is None: + description = "" + else: + description = self.description + + description = escape(description).replace("\n", Markup("
    ")) + return f"

    {description}

    " + + def get_body( + self, + environ: WSGIEnvironment | None = None, + scope: dict[str, t.Any] | None = None, + ) -> str: + """Get the HTML body.""" + return ( + "\n" + "\n" + f"{self.code} {escape(self.name)}\n" + f"

    {escape(self.name)}

    \n" + f"{self.get_description(environ)}\n" + ) + + def get_headers( + self, + environ: WSGIEnvironment | None = None, + scope: dict[str, t.Any] | None = None, + ) -> list[tuple[str, str]]: + """Get a list of headers.""" + return [("Content-Type", "text/html; charset=utf-8")] + + @t.overload + def get_response( + self, + environ: WSGIEnvironment | WSGIRequest | None = ..., + scope: None = None, + ) -> WSGIResponse: ... + @t.overload + def get_response( + self, + environ: None = None, + scope: dict[str, t.Any] = ..., + ) -> SansIOResponse: ... + def get_response( + self, + environ: WSGIEnvironment | WSGIRequest | None = None, + scope: dict[str, t.Any] | None = None, + ) -> WSGIResponse | SansIOResponse: + """Get a response object. + + :param environ: A WSGI environ dict or request object. If given, may be + used to customize the response based on the request. + :param scope: An ASGI scope dict. If given, may be used to customize the + response based on the request. + :return: A WSGI :class:`werkzeug.wrappers.Response` if called without + arguments or with ``environ``. A sans-IO + :class:`werkzeug.sansio.Response` for ASGI if called with + ``scope``. + """ + from .wrappers.response import Response + + if self.response is not None: + return self.response + if environ is not None: + environ = _get_environ(environ) + headers = self.get_headers(environ, scope) + return Response(self.get_body(environ, scope), self.code, headers) + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + """Call the exception as WSGI application. + + :param environ: the WSGI environment. + :param start_response: the response callable provided by the WSGI + server. + """ + response = self.get_response(environ) + return response(environ, start_response) + + def __str__(self) -> str: + code = self.code if self.code is not None else "???" + return f"{code} {self.name}: {self.description}" + + def __repr__(self) -> str: + code = self.code if self.code is not None else "???" + return f"<{type(self).__name__} '{code}: {self.name}'>" + + +class BadRequest(HTTPException): + """*400* `Bad Request` + + Raise if the browser sends something to the application the application + or server cannot handle. + """ + + code = 400 + description = ( + "The browser (or proxy) sent a request that this server could not understand." + ) + + +class BadRequestKeyError(BadRequest, KeyError): + """An exception that is used to signal both a :exc:`KeyError` and a + :exc:`BadRequest`. Used by many of the datastructures. + """ + + _description = BadRequest.description + #: Show the KeyError along with the HTTP error message in the + #: response. This should be disabled in production, but can be + #: useful in a debug mode. + show_exception = False + + def __init__(self, arg: object | None = None, *args: t.Any, **kwargs: t.Any): + super().__init__(*args, **kwargs) + + if arg is None: + KeyError.__init__(self) + else: + KeyError.__init__(self, arg) + + @property + def description(self) -> str: + if self.show_exception: + return f"{self._description}\n{KeyError.__name__}: {KeyError.__str__(self)}" + + return self._description + + @description.setter + def description(self, value: str) -> None: + self._description = value + + +class ClientDisconnected(BadRequest): + """Internal exception that is raised if Werkzeug detects a disconnected + client. Since the client is already gone at that point attempting to + send the error message to the client might not work and might ultimately + result in another exception in the server. Mainly this is here so that + it is silenced by default as far as Werkzeug is concerned. + + Since disconnections cannot be reliably detected and are unspecified + by WSGI to a large extent this might or might not be raised if a client + is gone. + + .. versionadded:: 0.8 + """ + + +class SecurityError(BadRequest): + """Raised if something triggers a security error. This is otherwise + exactly like a bad request error. + + .. versionadded:: 0.9 + """ + + +class BadHost(BadRequest): + """Raised if the submitted host is badly formatted. + + .. versionadded:: 0.11.2 + """ + + +class Unauthorized(HTTPException): + """*401* ``Unauthorized`` + + Raise if the user is not authorized to access a resource. + + The ``www_authenticate`` argument should be used to set the + ``WWW-Authenticate`` header. This is used for HTTP basic auth and + other schemes. Use :class:`~werkzeug.datastructures.WWWAuthenticate` + to create correctly formatted values. Strictly speaking a 401 + response is invalid if it doesn't provide at least one value for + this header, although real clients typically don't care. + + :param description: Override the default message used for the body + of the response. + :param www-authenticate: A single value, or list of values, for the + WWW-Authenticate header(s). + + .. versionchanged:: 2.0 + Serialize multiple ``www_authenticate`` items into multiple + ``WWW-Authenticate`` headers, rather than joining them + into a single value, for better interoperability. + + .. versionchanged:: 0.15.3 + If the ``www_authenticate`` argument is not set, the + ``WWW-Authenticate`` header is not set. + + .. versionchanged:: 0.15.3 + The ``response`` argument was restored. + + .. versionchanged:: 0.15.1 + ``description`` was moved back as the first argument, restoring + its previous position. + + .. versionchanged:: 0.15.0 + ``www_authenticate`` was added as the first argument, ahead of + ``description``. + """ + + code = 401 + description = ( + "The server could not verify that you are authorized to access" + " the URL requested. You either supplied the wrong credentials" + " (e.g. a bad password), or your browser doesn't understand" + " how to supply the credentials required." + ) + + def __init__( + self, + description: str | None = None, + response: SansIOResponse | None = None, + www_authenticate: None | (WWWAuthenticate | t.Iterable[WWWAuthenticate]) = None, + ) -> None: + super().__init__(description, response) + + from .datastructures import WWWAuthenticate + + if isinstance(www_authenticate, WWWAuthenticate): + www_authenticate = (www_authenticate,) + + self.www_authenticate = www_authenticate + + def get_headers( + self, + environ: WSGIEnvironment | None = None, + scope: dict[str, t.Any] | None = None, + ) -> list[tuple[str, str]]: + headers = super().get_headers(environ, scope) + if self.www_authenticate: + headers.extend(("WWW-Authenticate", str(x)) for x in self.www_authenticate) + return headers + + +class Forbidden(HTTPException): + """*403* `Forbidden` + + Raise if the user doesn't have the permission for the requested resource + but was authenticated. + """ + + code = 403 + description = ( + "You don't have the permission to access the requested" + " resource. It is either read-protected or not readable by the" + " server." + ) + + +class NotFound(HTTPException): + """*404* `Not Found` + + Raise if a resource does not exist and never existed. + """ + + code = 404 + description = ( + "The requested URL was not found on the server. If you entered" + " the URL manually please check your spelling and try again." + ) + + +class MethodNotAllowed(HTTPException): + """*405* `Method Not Allowed` + + Raise if the server used a method the resource does not handle. For + example `POST` if the resource is view only. Especially useful for REST. + + The first argument for this exception should be a list of allowed methods. + Strictly speaking the response would be invalid if you don't provide valid + methods in the header which you can do with that list. + """ + + code = 405 + description = "The method is not allowed for the requested URL." + + def __init__( + self, + valid_methods: t.Iterable[str] | None = None, + description: str | None = None, + response: SansIOResponse | None = None, + ) -> None: + """Takes an optional list of valid http methods + starting with werkzeug 0.3 the list will be mandatory.""" + super().__init__(description=description, response=response) + self.valid_methods = valid_methods + + def get_headers( + self, + environ: WSGIEnvironment | None = None, + scope: dict[str, t.Any] | None = None, + ) -> list[tuple[str, str]]: + headers = super().get_headers(environ, scope) + if self.valid_methods: + headers.append(("Allow", ", ".join(self.valid_methods))) + return headers + + +class NotAcceptable(HTTPException): + """*406* `Not Acceptable` + + Raise if the server can't return any content conforming to the + `Accept` headers of the client. + """ + + code = 406 + description = ( + "The resource identified by the request is only capable of" + " generating response entities which have content" + " characteristics not acceptable according to the accept" + " headers sent in the request." + ) + + +class RequestTimeout(HTTPException): + """*408* `Request Timeout` + + Raise to signalize a timeout. + """ + + code = 408 + description = ( + "The server closed the network connection because the browser" + " didn't finish the request within the specified time." + ) + + +class Conflict(HTTPException): + """*409* `Conflict` + + Raise to signal that a request cannot be completed because it conflicts + with the current state on the server. + + .. versionadded:: 0.7 + """ + + code = 409 + description = ( + "A conflict happened while processing the request. The" + " resource might have been modified while the request was being" + " processed." + ) + + +class Gone(HTTPException): + """*410* `Gone` + + Raise if a resource existed previously and went away without new location. + """ + + code = 410 + description = ( + "The requested URL is no longer available on this server and" + " there is no forwarding address. If you followed a link from a" + " foreign page, please contact the author of this page." + ) + + +class LengthRequired(HTTPException): + """*411* `Length Required` + + Raise if the browser submitted data but no ``Content-Length`` header which + is required for the kind of processing the server does. + """ + + code = 411 + description = ( + "A request with this method requires a valid Content-" + "Length header." + ) + + +class PreconditionFailed(HTTPException): + """*412* `Precondition Failed` + + Status code used in combination with ``If-Match``, ``If-None-Match``, or + ``If-Unmodified-Since``. + """ + + code = 412 + description = ( + "The precondition on the request for the URL failed positive evaluation." + ) + + +class RequestEntityTooLarge(HTTPException): + """*413* `Request Entity Too Large` + + The status code one should return if the data submitted exceeded a given + limit. + """ + + code = 413 + description = "The data value transmitted exceeds the capacity limit." + + +class RequestURITooLarge(HTTPException): + """*414* `Request URI Too Large` + + Like *413* but for too long URLs. + """ + + code = 414 + description = ( + "The length of the requested URL exceeds the capacity limit for" + " this server. The request cannot be processed." + ) + + +class UnsupportedMediaType(HTTPException): + """*415* `Unsupported Media Type` + + The status code returned if the server is unable to handle the media type + the client transmitted. + """ + + code = 415 + description = ( + "The server does not support the media type transmitted in the request." + ) + + +class RequestedRangeNotSatisfiable(HTTPException): + """*416* `Requested Range Not Satisfiable` + + The client asked for an invalid part of the file. + + .. versionadded:: 0.7 + """ + + code = 416 + description = "The server cannot provide the requested range." + + def __init__( + self, + length: int | None = None, + units: str = "bytes", + description: str | None = None, + response: SansIOResponse | None = None, + ) -> None: + """Takes an optional `Content-Range` header value based on ``length`` + parameter. + """ + super().__init__(description=description, response=response) + self.length = length + self.units = units + + def get_headers( + self, + environ: WSGIEnvironment | None = None, + scope: dict[str, t.Any] | None = None, + ) -> list[tuple[str, str]]: + headers = super().get_headers(environ, scope) + if self.length is not None: + headers.append(("Content-Range", f"{self.units} */{self.length}")) + return headers + + +class ExpectationFailed(HTTPException): + """*417* `Expectation Failed` + + The server cannot meet the requirements of the Expect request-header. + + .. versionadded:: 0.7 + """ + + code = 417 + description = "The server could not meet the requirements of the Expect header" + + +class ImATeapot(HTTPException): + """*418* `I'm a teapot` + + The server should return this if it is a teapot and someone attempted + to brew coffee with it. + + .. versionadded:: 0.7 + """ + + code = 418 + description = "This server is a teapot, not a coffee machine" + + +class MisdirectedRequest(HTTPException): + """421 Misdirected Request + + Indicates that the request was directed to a server that is not able to + produce a response. + + .. versionadded:: 3.1 + """ + + code = 421 + description = "The server is not able to produce a response." + + +class UnprocessableEntity(HTTPException): + """*422* `Unprocessable Entity` + + Used if the request is well formed, but the instructions are otherwise + incorrect. + """ + + code = 422 + description = ( + "The request was well-formed but was unable to be followed due" + " to semantic errors." + ) + + +class Locked(HTTPException): + """*423* `Locked` + + Used if the resource that is being accessed is locked. + """ + + code = 423 + description = "The resource that is being accessed is locked." + + +class FailedDependency(HTTPException): + """*424* `Failed Dependency` + + Used if the method could not be performed on the resource + because the requested action depended on another action and that action failed. + """ + + code = 424 + description = ( + "The method could not be performed on the resource because the" + " requested action depended on another action and that action" + " failed." + ) + + +class PreconditionRequired(HTTPException): + """*428* `Precondition Required` + + The server requires this request to be conditional, typically to prevent + the lost update problem, which is a race condition between two or more + clients attempting to update a resource through PUT or DELETE. By requiring + each client to include a conditional header ("If-Match" or "If-Unmodified- + Since") with the proper value retained from a recent GET request, the + server ensures that each client has at least seen the previous revision of + the resource. + """ + + code = 428 + description = ( + "This request is required to be conditional; try using" + ' "If-Match" or "If-Unmodified-Since".' + ) + + +class _RetryAfter(HTTPException): + """Adds an optional ``retry_after`` parameter which will set the + ``Retry-After`` header. May be an :class:`int` number of seconds or + a :class:`~datetime.datetime`. + """ + + def __init__( + self, + description: str | None = None, + response: SansIOResponse | None = None, + retry_after: datetime | int | None = None, + ) -> None: + super().__init__(description, response) + self.retry_after = retry_after + + def get_headers( + self, + environ: WSGIEnvironment | None = None, + scope: dict[str, t.Any] | None = None, + ) -> list[tuple[str, str]]: + headers = super().get_headers(environ, scope) + + if self.retry_after: + if isinstance(self.retry_after, datetime): + from .http import http_date + + value = http_date(self.retry_after) + else: + value = str(self.retry_after) + + headers.append(("Retry-After", value)) + + return headers + + +class TooManyRequests(_RetryAfter): + """*429* `Too Many Requests` + + The server is limiting the rate at which this user receives + responses, and this request exceeds that rate. (The server may use + any convenient method to identify users and their request rates). + The server may include a "Retry-After" header to indicate how long + the user should wait before retrying. + + :param retry_after: If given, set the ``Retry-After`` header to this + value. May be an :class:`int` number of seconds or a + :class:`~datetime.datetime`. + + .. versionchanged:: 1.0 + Added ``retry_after`` parameter. + """ + + code = 429 + description = "This user has exceeded an allotted request count. Try again later." + + +class RequestHeaderFieldsTooLarge(HTTPException): + """*431* `Request Header Fields Too Large` + + The server refuses to process the request because the header fields are too + large. One or more individual fields may be too large, or the set of all + headers is too large. + """ + + code = 431 + description = "One or more header fields exceeds the maximum size." + + +class UnavailableForLegalReasons(HTTPException): + """*451* `Unavailable For Legal Reasons` + + This status code indicates that the server is denying access to the + resource as a consequence of a legal demand. + """ + + code = 451 + description = "Unavailable for legal reasons." + + +class InternalServerError(HTTPException): + """*500* `Internal Server Error` + + Raise if an internal server error occurred. This is a good fallback if an + unknown error occurred in the dispatcher. + + .. versionchanged:: 1.0.0 + Added the :attr:`original_exception` attribute. + """ + + code = 500 + description = ( + "The server encountered an internal error and was unable to" + " complete your request. Either the server is overloaded or" + " there is an error in the application." + ) + + def __init__( + self, + description: str | None = None, + response: SansIOResponse | None = None, + original_exception: BaseException | None = None, + ) -> None: + #: The original exception that caused this 500 error. Can be + #: used by frameworks to provide context when handling + #: unexpected errors. + self.original_exception = original_exception + super().__init__(description=description, response=response) + + +class NotImplemented(HTTPException): + """*501* `Not Implemented` + + Raise if the application does not support the action requested by the + browser. + """ + + code = 501 + description = "The server does not support the action requested by the browser." + + +class BadGateway(HTTPException): + """*502* `Bad Gateway` + + If you do proxying in your application you should return this status code + if you received an invalid response from the upstream server it accessed + in attempting to fulfill the request. + """ + + code = 502 + description = ( + "The proxy server received an invalid response from an upstream server." + ) + + +class ServiceUnavailable(_RetryAfter): + """*503* `Service Unavailable` + + Status code you should return if a service is temporarily + unavailable. + + :param retry_after: If given, set the ``Retry-After`` header to this + value. May be an :class:`int` number of seconds or a + :class:`~datetime.datetime`. + + .. versionchanged:: 1.0 + Added ``retry_after`` parameter. + """ + + code = 503 + description = ( + "The server is temporarily unable to service your request due" + " to maintenance downtime or capacity problems. Please try" + " again later." + ) + + +class GatewayTimeout(HTTPException): + """*504* `Gateway Timeout` + + Status code you should return if a connection to an upstream server + times out. + """ + + code = 504 + description = "The connection to an upstream server timed out." + + +class HTTPVersionNotSupported(HTTPException): + """*505* `HTTP Version Not Supported` + + The server does not support the HTTP protocol version used in the request. + """ + + code = 505 + description = ( + "The server does not support the HTTP protocol version used in the request." + ) + + +default_exceptions: dict[int, type[HTTPException]] = {} + + +def _find_exceptions() -> None: + for obj in globals().values(): + try: + is_http_exception = issubclass(obj, HTTPException) + except TypeError: + is_http_exception = False + if not is_http_exception or obj.code is None: + continue + old_obj = default_exceptions.get(obj.code, None) + if old_obj is not None and issubclass(obj, old_obj): + continue + default_exceptions[obj.code] = obj + + +_find_exceptions() +del _find_exceptions + + +class Aborter: + """When passed a dict of code -> exception items it can be used as + callable that raises exceptions. If the first argument to the + callable is an integer it will be looked up in the mapping, if it's + a WSGI application it will be raised in a proxy exception. + + The rest of the arguments are forwarded to the exception constructor. + """ + + def __init__( + self, + mapping: dict[int, type[HTTPException]] | None = None, + extra: dict[int, type[HTTPException]] | None = None, + ) -> None: + if mapping is None: + mapping = default_exceptions + self.mapping = dict(mapping) + if extra is not None: + self.mapping.update(extra) + + def __call__( + self, code: int | SansIOResponse, *args: t.Any, **kwargs: t.Any + ) -> t.NoReturn: + from .sansio.response import Response + + if isinstance(code, Response): + raise HTTPException(response=code) + + if code not in self.mapping: + raise LookupError(f"no exception for {code!r}") + + raise self.mapping[code](*args, **kwargs) + + +def abort(status: int | SansIOResponse, *args: t.Any, **kwargs: t.Any) -> t.NoReturn: + """Raises an :py:exc:`HTTPException` for the given status code or WSGI + application. + + If a status code is given, it will be looked up in the list of + exceptions and will raise that exception. If passed a WSGI application, + it will wrap it in a proxy WSGI exception and raise that:: + + abort(404) # 404 Not Found + abort(Response('Hello World')) + + """ + _aborter(status, *args, **kwargs) + + +_aborter: Aborter = Aborter() diff --git a/lib/python3.12/site-packages/werkzeug/formparser.py b/lib/python3.12/site-packages/werkzeug/formparser.py new file mode 100644 index 0000000000000000000000000000000000000000..0103414976e0ec04afdc90bc08224b4db4773bfc --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/formparser.py @@ -0,0 +1,430 @@ +from __future__ import annotations + +import typing as t +from io import BytesIO +from urllib.parse import parse_qsl + +from ._internal import _plain_int +from .datastructures import FileStorage +from .datastructures import Headers +from .datastructures import MultiDict +from .exceptions import RequestEntityTooLarge +from .http import parse_options_header +from .sansio.multipart import Data +from .sansio.multipart import Epilogue +from .sansio.multipart import Field +from .sansio.multipart import File +from .sansio.multipart import MultipartDecoder +from .sansio.multipart import NeedData +from .wsgi import get_content_length +from .wsgi import get_input_stream + +# there are some platforms where SpooledTemporaryFile is not available. +# In that case we need to provide a fallback. +try: + from tempfile import SpooledTemporaryFile +except ImportError: + from tempfile import TemporaryFile + + SpooledTemporaryFile = None # type: ignore + +if t.TYPE_CHECKING: + import typing as te + + from _typeshed.wsgi import WSGIEnvironment + + t_parse_result = tuple[ + t.IO[bytes], MultiDict[str, str], MultiDict[str, FileStorage] + ] + + class TStreamFactory(te.Protocol): + def __call__( + self, + total_content_length: int | None, + content_type: str | None, + filename: str | None, + content_length: int | None = None, + ) -> t.IO[bytes]: ... + + +F = t.TypeVar("F", bound=t.Callable[..., t.Any]) + + +def default_stream_factory( + total_content_length: int | None, + content_type: str | None, + filename: str | None, + content_length: int | None = None, +) -> t.IO[bytes]: + max_size = 1024 * 500 + + if SpooledTemporaryFile is not None: + return t.cast(t.IO[bytes], SpooledTemporaryFile(max_size=max_size, mode="rb+")) + elif total_content_length is None or total_content_length > max_size: + return t.cast(t.IO[bytes], TemporaryFile("rb+")) + + return BytesIO() + + +def parse_form_data( + environ: WSGIEnvironment, + stream_factory: TStreamFactory | None = None, + max_form_memory_size: int | None = None, + max_content_length: int | None = None, + cls: type[MultiDict[str, t.Any]] | None = None, + silent: bool = True, + *, + max_form_parts: int | None = None, +) -> t_parse_result: + """Parse the form data in the environ and return it as tuple in the form + ``(stream, form, files)``. You should only call this method if the + transport method is `POST`, `PUT`, or `PATCH`. + + If the mimetype of the data transmitted is `multipart/form-data` the + files multidict will be filled with `FileStorage` objects. If the + mimetype is unknown the input stream is wrapped and returned as first + argument, else the stream is empty. + + This is a shortcut for the common usage of :class:`FormDataParser`. + + :param environ: the WSGI environment to be used for parsing. + :param stream_factory: An optional callable that returns a new read and + writeable file descriptor. This callable works + the same as :meth:`Response._get_file_stream`. + :param max_form_memory_size: the maximum number of bytes to be accepted for + in-memory stored form data. If the data + exceeds the value specified an + :exc:`~exceptions.RequestEntityTooLarge` + exception is raised. + :param max_content_length: If this is provided and the transmitted data + is longer than this value an + :exc:`~exceptions.RequestEntityTooLarge` + exception is raised. + :param cls: an optional dict class to use. If this is not specified + or `None` the default :class:`MultiDict` is used. + :param silent: If set to False parsing errors will not be caught. + :param max_form_parts: The maximum number of multipart parts to be parsed. If this + is exceeded, a :exc:`~exceptions.RequestEntityTooLarge` exception is raised. + :return: A tuple in the form ``(stream, form, files)``. + + .. versionchanged:: 3.0 + The ``charset`` and ``errors`` parameters were removed. + + .. versionchanged:: 2.3 + Added the ``max_form_parts`` parameter. + + .. versionadded:: 0.5.1 + Added the ``silent`` parameter. + + .. versionadded:: 0.5 + Added the ``max_form_memory_size``, ``max_content_length``, and ``cls`` + parameters. + """ + return FormDataParser( + stream_factory=stream_factory, + max_form_memory_size=max_form_memory_size, + max_content_length=max_content_length, + max_form_parts=max_form_parts, + silent=silent, + cls=cls, + ).parse_from_environ(environ) + + +class FormDataParser: + """This class implements parsing of form data for Werkzeug. By itself + it can parse multipart and url encoded form data. It can be subclassed + and extended but for most mimetypes it is a better idea to use the + untouched stream and expose it as separate attributes on a request + object. + + :param stream_factory: An optional callable that returns a new read and + writeable file descriptor. This callable works + the same as :meth:`Response._get_file_stream`. + :param max_form_memory_size: the maximum number of bytes to be accepted for + in-memory stored form data. If the data + exceeds the value specified an + :exc:`~exceptions.RequestEntityTooLarge` + exception is raised. + :param max_content_length: If this is provided and the transmitted data + is longer than this value an + :exc:`~exceptions.RequestEntityTooLarge` + exception is raised. + :param cls: an optional dict class to use. If this is not specified + or `None` the default :class:`MultiDict` is used. + :param silent: If set to False parsing errors will not be caught. + :param max_form_parts: The maximum number of multipart parts to be parsed. If this + is exceeded, a :exc:`~exceptions.RequestEntityTooLarge` exception is raised. + + .. versionchanged:: 3.0 + The ``charset`` and ``errors`` parameters were removed. + + .. versionchanged:: 3.0 + The ``parse_functions`` attribute and ``get_parse_func`` methods were removed. + + .. versionchanged:: 2.2.3 + Added the ``max_form_parts`` parameter. + + .. versionadded:: 0.8 + """ + + def __init__( + self, + stream_factory: TStreamFactory | None = None, + max_form_memory_size: int | None = None, + max_content_length: int | None = None, + cls: type[MultiDict[str, t.Any]] | None = None, + silent: bool = True, + *, + max_form_parts: int | None = None, + ) -> None: + if stream_factory is None: + stream_factory = default_stream_factory + + self.stream_factory = stream_factory + self.max_form_memory_size = max_form_memory_size + self.max_content_length = max_content_length + self.max_form_parts = max_form_parts + + if cls is None: + cls = t.cast("type[MultiDict[str, t.Any]]", MultiDict) + + self.cls = cls + self.silent = silent + + def parse_from_environ(self, environ: WSGIEnvironment) -> t_parse_result: + """Parses the information from the environment as form data. + + :param environ: the WSGI environment to be used for parsing. + :return: A tuple in the form ``(stream, form, files)``. + """ + stream = get_input_stream(environ, max_content_length=self.max_content_length) + content_length = get_content_length(environ) + mimetype, options = parse_options_header(environ.get("CONTENT_TYPE")) + return self.parse( + stream, + content_length=content_length, + mimetype=mimetype, + options=options, + ) + + def parse( + self, + stream: t.IO[bytes], + mimetype: str, + content_length: int | None, + options: dict[str, str] | None = None, + ) -> t_parse_result: + """Parses the information from the given stream, mimetype, + content length and mimetype parameters. + + :param stream: an input stream + :param mimetype: the mimetype of the data + :param content_length: the content length of the incoming data + :param options: optional mimetype parameters (used for + the multipart boundary for instance) + :return: A tuple in the form ``(stream, form, files)``. + + .. versionchanged:: 3.0 + The invalid ``application/x-url-encoded`` content type is not + treated as ``application/x-www-form-urlencoded``. + """ + if mimetype == "multipart/form-data": + parse_func = self._parse_multipart + elif mimetype == "application/x-www-form-urlencoded": + parse_func = self._parse_urlencoded + else: + return stream, self.cls(), self.cls() + + if options is None: + options = {} + + try: + return parse_func(stream, mimetype, content_length, options) + except ValueError: + if not self.silent: + raise + + return stream, self.cls(), self.cls() + + def _parse_multipart( + self, + stream: t.IO[bytes], + mimetype: str, + content_length: int | None, + options: dict[str, str], + ) -> t_parse_result: + parser = MultiPartParser( + stream_factory=self.stream_factory, + max_form_memory_size=self.max_form_memory_size, + max_form_parts=self.max_form_parts, + cls=self.cls, + ) + boundary = options.get("boundary", "").encode("ascii") + + if not boundary: + raise ValueError("Missing boundary") + + form, files = parser.parse(stream, boundary, content_length) + return stream, form, files + + def _parse_urlencoded( + self, + stream: t.IO[bytes], + mimetype: str, + content_length: int | None, + options: dict[str, str], + ) -> t_parse_result: + if ( + self.max_form_memory_size is not None + and content_length is not None + and content_length > self.max_form_memory_size + ): + raise RequestEntityTooLarge() + + items = parse_qsl( + stream.read().decode(), + keep_blank_values=True, + errors="werkzeug.url_quote", + ) + return stream, self.cls(items), self.cls() + + +class MultiPartParser: + def __init__( + self, + stream_factory: TStreamFactory | None = None, + max_form_memory_size: int | None = None, + cls: type[MultiDict[str, t.Any]] | None = None, + buffer_size: int = 64 * 1024, + max_form_parts: int | None = None, + ) -> None: + self.max_form_memory_size = max_form_memory_size + self.max_form_parts = max_form_parts + + if stream_factory is None: + stream_factory = default_stream_factory + + self.stream_factory = stream_factory + + if cls is None: + cls = t.cast("type[MultiDict[str, t.Any]]", MultiDict) + + self.cls = cls + self.buffer_size = buffer_size + + def fail(self, message: str) -> te.NoReturn: + raise ValueError(message) + + def get_part_charset(self, headers: Headers) -> str: + # Figure out input charset for current part + content_type = headers.get("content-type") + + if content_type: + parameters = parse_options_header(content_type)[1] + ct_charset = parameters.get("charset", "").lower() + + # A safe list of encodings. Modern clients should only send ASCII or UTF-8. + # This list will not be extended further. + if ct_charset in {"ascii", "us-ascii", "utf-8", "iso-8859-1"}: + return ct_charset + + return "utf-8" + + def start_file_streaming( + self, event: File, total_content_length: int | None + ) -> t.IO[bytes]: + content_type = event.headers.get("content-type") + + try: + content_length = _plain_int(event.headers["content-length"]) + except (KeyError, ValueError): + content_length = 0 + + container = self.stream_factory( + total_content_length=total_content_length, + filename=event.filename, + content_type=content_type, + content_length=content_length, + ) + return container + + def parse( + self, stream: t.IO[bytes], boundary: bytes, content_length: int | None + ) -> tuple[MultiDict[str, str], MultiDict[str, FileStorage]]: + current_part: Field | File + field_size: int | None = None + container: t.IO[bytes] | list[bytes] + _write: t.Callable[[bytes], t.Any] + + parser = MultipartDecoder( + boundary, + max_form_memory_size=self.max_form_memory_size, + max_parts=self.max_form_parts, + ) + + fields = [] + files = [] + + for data in _chunk_iter(stream.read, self.buffer_size): + parser.receive_data(data) + event = parser.next_event() + while not isinstance(event, (Epilogue, NeedData)): + if isinstance(event, Field): + current_part = event + field_size = 0 + container = [] + _write = container.append + elif isinstance(event, File): + current_part = event + field_size = None + container = self.start_file_streaming(event, content_length) + _write = container.write + elif isinstance(event, Data): + if self.max_form_memory_size is not None and field_size is not None: + # Ensure that accumulated data events do not exceed limit. + # Also checked within single event in MultipartDecoder. + field_size += len(event.data) + + if field_size > self.max_form_memory_size: + raise RequestEntityTooLarge() + + _write(event.data) + if not event.more_data: + if isinstance(current_part, Field): + value = b"".join(container).decode( + self.get_part_charset(current_part.headers), "replace" + ) + fields.append((current_part.name, value)) + else: + container = t.cast(t.IO[bytes], container) + container.seek(0) + files.append( + ( + current_part.name, + FileStorage( + container, + current_part.filename, + current_part.name, + headers=current_part.headers, + ), + ) + ) + + event = parser.next_event() + + return self.cls(fields), self.cls(files) + + +def _chunk_iter(read: t.Callable[[int], bytes], size: int) -> t.Iterator[bytes | None]: + """Read data in chunks for multipart/form-data parsing. Stop if no data is read. + Yield ``None`` at the end to signal end of parsing. + """ + while True: + data = read(size) + + if not data: + break + + yield data + + yield None diff --git a/lib/python3.12/site-packages/werkzeug/http.py b/lib/python3.12/site-packages/werkzeug/http.py new file mode 100644 index 0000000000000000000000000000000000000000..aee4f59a8b7557b745f3bff247bc1063c62a5bf4 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/http.py @@ -0,0 +1,1405 @@ +from __future__ import annotations + +import email.utils +import re +import typing as t +import warnings +from datetime import date +from datetime import datetime +from datetime import time +from datetime import timedelta +from datetime import timezone +from enum import Enum +from hashlib import sha1 +from time import mktime +from time import struct_time +from urllib.parse import quote +from urllib.parse import unquote +from urllib.request import parse_http_list as _parse_list_header + +from ._internal import _dt_as_utc +from ._internal import _plain_int + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIEnvironment + +_token_chars = frozenset( + "!#$%&'*+-.0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ^_`abcdefghijklmnopqrstuvwxyz|~" +) +_etag_re = re.compile(r'([Ww]/)?(?:"(.*?)"|(.*?))(?:\s*,\s*|$)') +_entity_headers = frozenset( + [ + "allow", + "content-encoding", + "content-language", + "content-length", + "content-location", + "content-md5", + "content-range", + "content-type", + "expires", + "last-modified", + ] +) +_hop_by_hop_headers = frozenset( + [ + "connection", + "keep-alive", + "proxy-authenticate", + "proxy-authorization", + "te", + "trailer", + "transfer-encoding", + "upgrade", + ] +) +HTTP_STATUS_CODES = { + 100: "Continue", + 101: "Switching Protocols", + 102: "Processing", + 103: "Early Hints", # see RFC 8297 + 200: "OK", + 201: "Created", + 202: "Accepted", + 203: "Non Authoritative Information", + 204: "No Content", + 205: "Reset Content", + 206: "Partial Content", + 207: "Multi Status", + 208: "Already Reported", # see RFC 5842 + 226: "IM Used", # see RFC 3229 + 300: "Multiple Choices", + 301: "Moved Permanently", + 302: "Found", + 303: "See Other", + 304: "Not Modified", + 305: "Use Proxy", + 306: "Switch Proxy", # unused + 307: "Temporary Redirect", + 308: "Permanent Redirect", + 400: "Bad Request", + 401: "Unauthorized", + 402: "Payment Required", # unused + 403: "Forbidden", + 404: "Not Found", + 405: "Method Not Allowed", + 406: "Not Acceptable", + 407: "Proxy Authentication Required", + 408: "Request Timeout", + 409: "Conflict", + 410: "Gone", + 411: "Length Required", + 412: "Precondition Failed", + 413: "Request Entity Too Large", + 414: "Request URI Too Long", + 415: "Unsupported Media Type", + 416: "Requested Range Not Satisfiable", + 417: "Expectation Failed", + 418: "I'm a teapot", # see RFC 2324 + 421: "Misdirected Request", # see RFC 7540 + 422: "Unprocessable Entity", + 423: "Locked", + 424: "Failed Dependency", + 425: "Too Early", # see RFC 8470 + 426: "Upgrade Required", + 428: "Precondition Required", # see RFC 6585 + 429: "Too Many Requests", + 431: "Request Header Fields Too Large", + 449: "Retry With", # proprietary MS extension + 451: "Unavailable For Legal Reasons", + 500: "Internal Server Error", + 501: "Not Implemented", + 502: "Bad Gateway", + 503: "Service Unavailable", + 504: "Gateway Timeout", + 505: "HTTP Version Not Supported", + 506: "Variant Also Negotiates", # see RFC 2295 + 507: "Insufficient Storage", + 508: "Loop Detected", # see RFC 5842 + 510: "Not Extended", + 511: "Network Authentication Failed", +} + + +class COEP(Enum): + """Cross Origin Embedder Policies""" + + UNSAFE_NONE = "unsafe-none" + REQUIRE_CORP = "require-corp" + + +class COOP(Enum): + """Cross Origin Opener Policies""" + + UNSAFE_NONE = "unsafe-none" + SAME_ORIGIN_ALLOW_POPUPS = "same-origin-allow-popups" + SAME_ORIGIN = "same-origin" + + +def quote_header_value(value: t.Any, allow_token: bool = True) -> str: + """Add double quotes around a header value. If the header contains only ASCII token + characters, it will be returned unchanged. If the header contains ``"`` or ``\\`` + characters, they will be escaped with an additional ``\\`` character. + + This is the reverse of :func:`unquote_header_value`. + + :param value: The value to quote. Will be converted to a string. + :param allow_token: Disable to quote the value even if it only has token characters. + + .. versionchanged:: 3.0 + Passing bytes is not supported. + + .. versionchanged:: 3.0 + The ``extra_chars`` parameter is removed. + + .. versionchanged:: 2.3 + The value is quoted if it is the empty string. + + .. versionadded:: 0.5 + """ + value_str = str(value) + + if not value_str: + return '""' + + if allow_token: + token_chars = _token_chars + + if token_chars.issuperset(value_str): + return value_str + + value_str = value_str.replace("\\", "\\\\").replace('"', '\\"') + return f'"{value_str}"' + + +def unquote_header_value(value: str) -> str: + """Remove double quotes and decode slash-escaped ``"`` and ``\\`` characters in a + header value. + + This is the reverse of :func:`quote_header_value`. + + :param value: The header value to unquote. + + .. versionchanged:: 3.0 + The ``is_filename`` parameter is removed. + """ + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + return value.replace("\\\\", "\\").replace('\\"', '"') + + return value + + +def dump_options_header(header: str | None, options: t.Mapping[str, t.Any]) -> str: + """Produce a header value and ``key=value`` parameters separated by semicolons + ``;``. For example, the ``Content-Type`` header. + + .. code-block:: python + + dump_options_header("text/html", {"charset": "UTF-8"}) + 'text/html; charset=UTF-8' + + This is the reverse of :func:`parse_options_header`. + + If a value contains non-token characters, it will be quoted. + + If a value is ``None``, the parameter is skipped. + + In some keys for some headers, a UTF-8 value can be encoded using a special + ``key*=UTF-8''value`` form, where ``value`` is percent encoded. This function will + not produce that format automatically, but if a given key ends with an asterisk + ``*``, the value is assumed to have that form and will not be quoted further. + + :param header: The primary header value. + :param options: Parameters to encode as ``key=value`` pairs. + + .. versionchanged:: 2.3 + Keys with ``None`` values are skipped rather than treated as a bare key. + + .. versionchanged:: 2.2.3 + If a key ends with ``*``, its value will not be quoted. + """ + segments = [] + + if header is not None: + segments.append(header) + + for key, value in options.items(): + if value is None: + continue + + if key[-1] == "*": + segments.append(f"{key}={value}") + else: + segments.append(f"{key}={quote_header_value(value)}") + + return "; ".join(segments) + + +def dump_header(iterable: dict[str, t.Any] | t.Iterable[t.Any]) -> str: + """Produce a header value from a list of items or ``key=value`` pairs, separated by + commas ``,``. + + This is the reverse of :func:`parse_list_header`, :func:`parse_dict_header`, and + :func:`parse_set_header`. + + If a value contains non-token characters, it will be quoted. + + If a value is ``None``, the key is output alone. + + In some keys for some headers, a UTF-8 value can be encoded using a special + ``key*=UTF-8''value`` form, where ``value`` is percent encoded. This function will + not produce that format automatically, but if a given key ends with an asterisk + ``*``, the value is assumed to have that form and will not be quoted further. + + .. code-block:: python + + dump_header(["foo", "bar baz"]) + 'foo, "bar baz"' + + dump_header({"foo": "bar baz"}) + 'foo="bar baz"' + + :param iterable: The items to create a header from. + + .. versionchanged:: 3.0 + The ``allow_token`` parameter is removed. + + .. versionchanged:: 2.2.3 + If a key ends with ``*``, its value will not be quoted. + """ + if isinstance(iterable, dict): + items = [] + + for key, value in iterable.items(): + if value is None: + items.append(key) + elif key[-1] == "*": + items.append(f"{key}={value}") + else: + items.append(f"{key}={quote_header_value(value)}") + else: + items = [quote_header_value(x) for x in iterable] + + return ", ".join(items) + + +def dump_csp_header(header: ds.ContentSecurityPolicy) -> str: + """Dump a Content Security Policy header. + + These are structured into policies such as "default-src 'self'; + script-src 'self'". + + .. versionadded:: 1.0.0 + Support for Content Security Policy headers was added. + + """ + return "; ".join(f"{key} {value}" for key, value in header.items()) + + +def parse_list_header(value: str) -> list[str]: + """Parse a header value that consists of a list of comma separated items according + to `RFC 9110 `__. + + This extends :func:`urllib.request.parse_http_list` to remove surrounding quotes + from values. + + .. code-block:: python + + parse_list_header('token, "quoted value"') + ['token', 'quoted value'] + + This is the reverse of :func:`dump_header`. + + :param value: The header value to parse. + """ + result = [] + + for item in _parse_list_header(value): + if len(item) >= 2 and item[0] == item[-1] == '"': + item = item[1:-1] + + result.append(item) + + return result + + +def parse_dict_header(value: str) -> dict[str, str | None]: + """Parse a list header using :func:`parse_list_header`, then parse each item as a + ``key=value`` pair. + + .. code-block:: python + + parse_dict_header('a=b, c="d, e", f') + {"a": "b", "c": "d, e", "f": None} + + This is the reverse of :func:`dump_header`. + + If a key does not have a value, it is ``None``. + + This handles charsets for values as described in + `RFC 2231 `__. Only ASCII, UTF-8, + and ISO-8859-1 charsets are accepted, otherwise the value remains quoted. + + :param value: The header value to parse. + + .. versionchanged:: 3.0 + Passing bytes is not supported. + + .. versionchanged:: 3.0 + The ``cls`` argument is removed. + + .. versionchanged:: 2.3 + Added support for ``key*=charset''value`` encoded items. + + .. versionchanged:: 0.9 + The ``cls`` argument was added. + """ + result: dict[str, str | None] = {} + + for item in parse_list_header(value): + key, has_value, value = item.partition("=") + key = key.strip() + + if not key: + # =value is not valid + continue + + if not has_value: + result[key] = None + continue + + value = value.strip() + encoding: str | None = None + + if key[-1] == "*": + # key*=charset''value becomes key=value, where value is percent encoded + # adapted from parse_options_header, without the continuation handling + key = key[:-1] + match = _charset_value_re.match(value) + + if match: + # If there is a charset marker in the value, split it off. + encoding, value = match.groups() + encoding = encoding.lower() + + # A safe list of encodings. Modern clients should only send ASCII or UTF-8. + # This list will not be extended further. An invalid encoding will leave the + # value quoted. + if encoding in {"ascii", "us-ascii", "utf-8", "iso-8859-1"}: + # invalid bytes are replaced during unquoting + value = unquote(value, encoding=encoding) + + if len(value) >= 2 and value[0] == value[-1] == '"': + value = value[1:-1] + + result[key] = value + + return result + + +# https://httpwg.org/specs/rfc9110.html#parameter +_parameter_key_re = re.compile(r"([\w!#$%&'*+\-.^`|~]+)=", flags=re.ASCII) +_parameter_token_value_re = re.compile(r"[\w!#$%&'*+\-.^`|~]+", flags=re.ASCII) +# https://www.rfc-editor.org/rfc/rfc2231#section-4 +_charset_value_re = re.compile( + r""" + ([\w!#$%&*+\-.^`|~]*)' # charset part, could be empty + [\w!#$%&*+\-.^`|~]*' # don't care about language part, usually empty + ([\w!#$%&'*+\-.^`|~]+) # one or more token chars with percent encoding + """, + re.ASCII | re.VERBOSE, +) +# https://www.rfc-editor.org/rfc/rfc2231#section-3 +_continuation_re = re.compile(r"\*(\d+)$", re.ASCII) + + +def parse_options_header(value: str | None) -> tuple[str, dict[str, str]]: + """Parse a header that consists of a value with ``key=value`` parameters separated + by semicolons ``;``. For example, the ``Content-Type`` header. + + .. code-block:: python + + parse_options_header("text/html; charset=UTF-8") + ('text/html', {'charset': 'UTF-8'}) + + parse_options_header("") + ("", {}) + + This is the reverse of :func:`dump_options_header`. + + This parses valid parameter parts as described in + `RFC 9110 `__. Invalid parts are + skipped. + + This handles continuations and charsets as described in + `RFC 2231 `__, although not as + strictly as the RFC. Only ASCII, UTF-8, and ISO-8859-1 charsets are accepted, + otherwise the value remains quoted. + + Clients may not be consistent in how they handle a quote character within a quoted + value. The `HTML Standard `__ + replaces it with ``%22`` in multipart form data. + `RFC 9110 `__ uses backslash + escapes in HTTP headers. Both are decoded to the ``"`` character. + + Clients may not be consistent in how they handle non-ASCII characters. HTML + documents must declare ````, otherwise browsers may replace with + HTML character references, which can be decoded using :func:`html.unescape`. + + :param value: The header value to parse. + :return: ``(value, options)``, where ``options`` is a dict + + .. versionchanged:: 2.3 + Invalid parts, such as keys with no value, quoted keys, and incorrectly quoted + values, are discarded instead of treating as ``None``. + + .. versionchanged:: 2.3 + Only ASCII, UTF-8, and ISO-8859-1 are accepted for charset values. + + .. versionchanged:: 2.3 + Escaped quotes in quoted values, like ``%22`` and ``\\"``, are handled. + + .. versionchanged:: 2.2 + Option names are always converted to lowercase. + + .. versionchanged:: 2.2 + The ``multiple`` parameter was removed. + + .. versionchanged:: 0.15 + :rfc:`2231` parameter continuations are handled. + + .. versionadded:: 0.5 + """ + if value is None: + return "", {} + + value, _, rest = value.partition(";") + value = value.strip() + rest = rest.strip() + + if not value or not rest: + # empty (invalid) value, or value without options + return value, {} + + # Collect all valid key=value parts without processing the value. + parts: list[tuple[str, str]] = [] + + while True: + if (m := _parameter_key_re.match(rest)) is not None: + pk = m.group(1).lower() + rest = rest[m.end() :] + + # Value may be a token. + if (m := _parameter_token_value_re.match(rest)) is not None: + parts.append((pk, m.group())) + + # Value may be a quoted string, find the closing quote. + elif rest[:1] == '"': + pos = 1 + length = len(rest) + + while pos < length: + if rest[pos : pos + 2] in {"\\\\", '\\"'}: + # Consume escaped slashes and quotes. + pos += 2 + elif rest[pos] == '"': + # Stop at an unescaped quote. + parts.append((pk, rest[: pos + 1])) + rest = rest[pos + 1 :] + break + else: + # Consume any other character. + pos += 1 + + # Find the next section delimited by `;`, if any. + if (end := rest.find(";")) == -1: + break + + rest = rest[end + 1 :].lstrip() + + options: dict[str, str] = {} + encoding: str | None = None + continued_encoding: str | None = None + + # For each collected part, process optional charset and continuation, + # unquote quoted values. + for pk, pv in parts: + if pk[-1] == "*": + # key*=charset''value becomes key=value, where value is percent encoded + pk = pk[:-1] + match = _charset_value_re.match(pv) + + if match: + # If there is a valid charset marker in the value, split it off. + encoding, pv = match.groups() + # This might be the empty string, handled next. + encoding = encoding.lower() + + # No charset marker, or marker with empty charset value. + if not encoding: + encoding = continued_encoding + + # A safe list of encodings. Modern clients should only send ASCII or UTF-8. + # This list will not be extended further. An invalid encoding will leave the + # value quoted. + if encoding in {"ascii", "us-ascii", "utf-8", "iso-8859-1"}: + # Continuation parts don't require their own charset marker. This is + # looser than the RFC, it will persist across different keys and allows + # changing the charset during a continuation. But this implementation is + # much simpler than tracking the full state. + continued_encoding = encoding + # invalid bytes are replaced during unquoting + pv = unquote(pv, encoding=encoding) + + # Remove quotes. At this point the value cannot be empty or a single quote. + if pv[0] == pv[-1] == '"': + # HTTP headers use slash, multipart form data uses percent + pv = pv[1:-1].replace("\\\\", "\\").replace('\\"', '"').replace("%22", '"') + + match = _continuation_re.search(pk) + + if match: + # key*0=a; key*1=b becomes key=ab + pk = pk[: match.start()] + options[pk] = options.get(pk, "") + pv + else: + options[pk] = pv + + return value, options + + +_q_value_re = re.compile(r"-?\d+(\.\d+)?", re.ASCII) +_TAnyAccept = t.TypeVar("_TAnyAccept", bound="ds.Accept") + + +@t.overload +def parse_accept_header(value: str | None) -> ds.Accept: ... + + +@t.overload +def parse_accept_header(value: str | None, cls: type[_TAnyAccept]) -> _TAnyAccept: ... + + +def parse_accept_header( + value: str | None, cls: type[_TAnyAccept] | None = None +) -> _TAnyAccept: + """Parse an ``Accept`` header according to + `RFC 9110 `__. + + Returns an :class:`.Accept` instance, which can sort and inspect items based on + their quality parameter. When parsing ``Accept-Charset``, ``Accept-Encoding``, or + ``Accept-Language``, pass the appropriate :class:`.Accept` subclass. + + :param value: The header value to parse. + :param cls: The :class:`.Accept` class to wrap the result in. + :return: An instance of ``cls``. + + .. versionchanged:: 2.3 + Parse according to RFC 9110. Items with invalid ``q`` values are skipped. + """ + if cls is None: + cls = t.cast(type[_TAnyAccept], ds.Accept) + + if not value: + return cls(None) + + result = [] + + for item in parse_list_header(value): + item, options = parse_options_header(item) + + if "q" in options: + # pop q, remaining options are reconstructed + q_str = options.pop("q").strip() + + if _q_value_re.fullmatch(q_str) is None: + # ignore an invalid q + continue + + q = float(q_str) + + if q < 0 or q > 1: + # ignore an invalid q + continue + else: + q = 1 + + if options: + # reconstruct the media type with any options + item = dump_options_header(item, options) + + result.append((item, q)) + + return cls(result) + + +_TAnyCC = t.TypeVar("_TAnyCC", bound="ds.cache_control._CacheControl") + + +@t.overload +def parse_cache_control_header( + value: str | None, + on_update: t.Callable[[ds.cache_control._CacheControl], None] | None = None, +) -> ds.RequestCacheControl: ... + + +@t.overload +def parse_cache_control_header( + value: str | None, + on_update: t.Callable[[ds.cache_control._CacheControl], None] | None = None, + cls: type[_TAnyCC] = ..., +) -> _TAnyCC: ... + + +def parse_cache_control_header( + value: str | None, + on_update: t.Callable[[ds.cache_control._CacheControl], None] | None = None, + cls: type[_TAnyCC] | None = None, +) -> _TAnyCC: + """Parse a cache control header. The RFC differs between response and + request cache control, this method does not. It's your responsibility + to not use the wrong control statements. + + .. versionadded:: 0.5 + The `cls` was added. If not specified an immutable + :class:`~werkzeug.datastructures.RequestCacheControl` is returned. + + :param value: a cache control header to be parsed. + :param on_update: an optional callable that is called every time a value + on the :class:`~werkzeug.datastructures.CacheControl` + object is changed. + :param cls: the class for the returned object. By default + :class:`~werkzeug.datastructures.RequestCacheControl` is used. + :return: a `cls` object. + """ + if cls is None: + cls = t.cast("type[_TAnyCC]", ds.RequestCacheControl) + + if not value: + return cls((), on_update) + + return cls(parse_dict_header(value), on_update) + + +_TAnyCSP = t.TypeVar("_TAnyCSP", bound="ds.ContentSecurityPolicy") + + +@t.overload +def parse_csp_header( + value: str | None, + on_update: t.Callable[[ds.ContentSecurityPolicy], None] | None = None, +) -> ds.ContentSecurityPolicy: ... + + +@t.overload +def parse_csp_header( + value: str | None, + on_update: t.Callable[[ds.ContentSecurityPolicy], None] | None = None, + cls: type[_TAnyCSP] = ..., +) -> _TAnyCSP: ... + + +def parse_csp_header( + value: str | None, + on_update: t.Callable[[ds.ContentSecurityPolicy], None] | None = None, + cls: type[_TAnyCSP] | None = None, +) -> _TAnyCSP: + """Parse a Content Security Policy header. + + .. versionadded:: 1.0.0 + Support for Content Security Policy headers was added. + + :param value: a csp header to be parsed. + :param on_update: an optional callable that is called every time a value + on the object is changed. + :param cls: the class for the returned object. By default + :class:`~werkzeug.datastructures.ContentSecurityPolicy` is used. + :return: a `cls` object. + """ + if cls is None: + cls = t.cast("type[_TAnyCSP]", ds.ContentSecurityPolicy) + + if value is None: + return cls((), on_update) + + items = [] + + for policy in value.split(";"): + policy = policy.strip() + + # Ignore badly formatted policies (no space) + if " " in policy: + directive, value = policy.strip().split(" ", 1) + items.append((directive.strip(), value.strip())) + + return cls(items, on_update) + + +def parse_set_header( + value: str | None, + on_update: t.Callable[[ds.HeaderSet], None] | None = None, +) -> ds.HeaderSet: + """Parse a set-like header and return a + :class:`~werkzeug.datastructures.HeaderSet` object: + + >>> hs = parse_set_header('token, "quoted value"') + + The return value is an object that treats the items case-insensitively + and keeps the order of the items: + + >>> 'TOKEN' in hs + True + >>> hs.index('quoted value') + 1 + >>> hs + HeaderSet(['token', 'quoted value']) + + To create a header from the :class:`HeaderSet` again, use the + :func:`dump_header` function. + + :param value: a set header to be parsed. + :param on_update: an optional callable that is called every time a + value on the :class:`~werkzeug.datastructures.HeaderSet` + object is changed. + :return: a :class:`~werkzeug.datastructures.HeaderSet` + """ + if not value: + return ds.HeaderSet(None, on_update) + return ds.HeaderSet(parse_list_header(value), on_update) + + +def parse_if_range_header(value: str | None) -> ds.IfRange: + """Parses an if-range header which can be an etag or a date. Returns + a :class:`~werkzeug.datastructures.IfRange` object. + + .. versionchanged:: 2.0 + If the value represents a datetime, it is timezone-aware. + + .. versionadded:: 0.7 + """ + if not value: + return ds.IfRange() + date = parse_date(value) + if date is not None: + return ds.IfRange(date=date) + # drop weakness information + return ds.IfRange(unquote_etag(value)[0]) + + +def parse_range_header( + value: str | None, make_inclusive: bool = True +) -> ds.Range | None: + """Parses a range header into a :class:`~werkzeug.datastructures.Range` + object. If the header is missing or malformed `None` is returned. + `ranges` is a list of ``(start, stop)`` tuples where the ranges are + non-inclusive. + + .. versionadded:: 0.7 + """ + if not value or "=" not in value: + return None + + ranges = [] + last_end = 0 + units, rng = value.split("=", 1) + units = units.strip().lower() + + for item in rng.split(","): + item = item.strip() + if "-" not in item: + return None + if item.startswith("-"): + if last_end < 0: + return None + try: + begin = _plain_int(item) + except ValueError: + return None + end = None + last_end = -1 + elif "-" in item: + begin_str, end_str = item.split("-", 1) + begin_str = begin_str.strip() + end_str = end_str.strip() + + try: + begin = _plain_int(begin_str) + except ValueError: + return None + + if begin < last_end or last_end < 0: + return None + if end_str: + try: + end = _plain_int(end_str) + 1 + except ValueError: + return None + + if begin >= end: + return None + else: + end = None + last_end = end if end is not None else -1 + ranges.append((begin, end)) + + return ds.Range(units, ranges) + + +def parse_content_range_header( + value: str | None, + on_update: t.Callable[[ds.ContentRange], None] | None = None, +) -> ds.ContentRange | None: + """Parses a range header into a + :class:`~werkzeug.datastructures.ContentRange` object or `None` if + parsing is not possible. + + .. versionadded:: 0.7 + + :param value: a content range header to be parsed. + :param on_update: an optional callable that is called every time a value + on the :class:`~werkzeug.datastructures.ContentRange` + object is changed. + """ + if value is None: + return None + try: + units, rangedef = (value or "").strip().split(None, 1) + except ValueError: + return None + + if "/" not in rangedef: + return None + rng, length_str = rangedef.split("/", 1) + if length_str == "*": + length = None + else: + try: + length = _plain_int(length_str) + except ValueError: + return None + + if rng == "*": + if not is_byte_range_valid(None, None, length): + return None + + return ds.ContentRange(units, None, None, length, on_update=on_update) + elif "-" not in rng: + return None + + start_str, stop_str = rng.split("-", 1) + try: + start = _plain_int(start_str) + stop = _plain_int(stop_str) + 1 + except ValueError: + return None + + if is_byte_range_valid(start, stop, length): + return ds.ContentRange(units, start, stop, length, on_update=on_update) + + return None + + +def quote_etag(etag: str, weak: bool = False) -> str: + """Quote an etag. + + :param etag: the etag to quote. + :param weak: set to `True` to tag it "weak". + """ + if '"' in etag: + raise ValueError("invalid etag") + etag = f'"{etag}"' + if weak: + etag = f"W/{etag}" + return etag + + +@t.overload +def unquote_etag(etag: str) -> tuple[str, bool]: ... +@t.overload +def unquote_etag(etag: None) -> tuple[None, None]: ... +def unquote_etag( + etag: str | None, +) -> tuple[str, bool] | tuple[None, None]: + """Unquote a single etag: + + >>> unquote_etag('W/"bar"') + ('bar', True) + >>> unquote_etag('"bar"') + ('bar', False) + + :param etag: the etag identifier to unquote. + :return: a ``(etag, weak)`` tuple. + """ + if not etag: + return None, None + etag = etag.strip() + weak = False + if etag.startswith(("W/", "w/")): + weak = True + etag = etag[2:] + if etag[:1] == etag[-1:] == '"': + etag = etag[1:-1] + return etag, weak + + +def parse_etags(value: str | None) -> ds.ETags: + """Parse an etag header. + + :param value: the tag header to parse + :return: an :class:`~werkzeug.datastructures.ETags` object. + """ + if not value: + return ds.ETags() + strong = [] + weak = [] + end = len(value) + pos = 0 + while pos < end: + match = _etag_re.match(value, pos) + if match is None: + break + is_weak, quoted, raw = match.groups() + if raw == "*": + return ds.ETags(star_tag=True) + elif quoted: + raw = quoted + if is_weak: + weak.append(raw) + else: + strong.append(raw) + pos = match.end() + return ds.ETags(strong, weak) + + +def generate_etag(data: bytes) -> str: + """Generate an etag for some data. + + .. versionchanged:: 2.0 + Use SHA-1. MD5 may not be available in some environments. + """ + return sha1(data).hexdigest() + + +def parse_date(value: str | None) -> datetime | None: + """Parse an :rfc:`2822` date into a timezone-aware + :class:`datetime.datetime` object, or ``None`` if parsing fails. + + This is a wrapper for :func:`email.utils.parsedate_to_datetime`. It + returns ``None`` if parsing fails instead of raising an exception, + and always returns a timezone-aware datetime object. If the string + doesn't have timezone information, it is assumed to be UTC. + + :param value: A string with a supported date format. + + .. versionchanged:: 2.0 + Return a timezone-aware datetime object. Use + ``email.utils.parsedate_to_datetime``. + """ + if value is None: + return None + + try: + dt = email.utils.parsedate_to_datetime(value) + except (TypeError, ValueError): + return None + + if dt.tzinfo is None: + return dt.replace(tzinfo=timezone.utc) + + return dt + + +def http_date( + timestamp: datetime | date | int | float | struct_time | None = None, +) -> str: + """Format a datetime object or timestamp into an :rfc:`2822` date + string. + + This is a wrapper for :func:`email.utils.format_datetime`. It + assumes naive datetime objects are in UTC instead of raising an + exception. + + :param timestamp: The datetime or timestamp to format. Defaults to + the current time. + + .. versionchanged:: 2.0 + Use ``email.utils.format_datetime``. Accept ``date`` objects. + """ + if isinstance(timestamp, date): + if not isinstance(timestamp, datetime): + # Assume plain date is midnight UTC. + timestamp = datetime.combine(timestamp, time(), tzinfo=timezone.utc) + else: + # Ensure datetime is timezone-aware. + timestamp = _dt_as_utc(timestamp) + + return email.utils.format_datetime(timestamp, usegmt=True) + + if isinstance(timestamp, struct_time): + timestamp = mktime(timestamp) + + return email.utils.formatdate(timestamp, usegmt=True) + + +def parse_age(value: str | None = None) -> timedelta | None: + """Parses a base-10 integer count of seconds into a timedelta. + + If parsing fails, the return value is `None`. + + :param value: a string consisting of an integer represented in base-10 + :return: a :class:`datetime.timedelta` object or `None`. + """ + if not value: + return None + try: + seconds = int(value) + except ValueError: + return None + if seconds < 0: + return None + try: + return timedelta(seconds=seconds) + except OverflowError: + return None + + +def dump_age(age: timedelta | int | None = None) -> str | None: + """Formats the duration as a base-10 integer. + + :param age: should be an integer number of seconds, + a :class:`datetime.timedelta` object, or, + if the age is unknown, `None` (default). + """ + if age is None: + return None + if isinstance(age, timedelta): + age = int(age.total_seconds()) + else: + age = int(age) + + if age < 0: + raise ValueError("age cannot be negative") + + return str(age) + + +def is_resource_modified( + environ: WSGIEnvironment, + etag: str | None = None, + data: bytes | None = None, + last_modified: datetime | str | None = None, + ignore_if_range: bool = True, +) -> bool: + """Convenience method for conditional requests. + + :param environ: the WSGI environment of the request to be checked. + :param etag: the etag for the response for comparison. + :param data: or alternatively the data of the response to automatically + generate an etag using :func:`generate_etag`. + :param last_modified: an optional date of the last modification. + :param ignore_if_range: If `False`, `If-Range` header will be taken into + account. + :return: `True` if the resource was modified, otherwise `False`. + + .. versionchanged:: 2.0 + SHA-1 is used to generate an etag value for the data. MD5 may + not be available in some environments. + + .. versionchanged:: 1.0.0 + The check is run for methods other than ``GET`` and ``HEAD``. + """ + return _sansio_http.is_resource_modified( + http_range=environ.get("HTTP_RANGE"), + http_if_range=environ.get("HTTP_IF_RANGE"), + http_if_modified_since=environ.get("HTTP_IF_MODIFIED_SINCE"), + http_if_none_match=environ.get("HTTP_IF_NONE_MATCH"), + http_if_match=environ.get("HTTP_IF_MATCH"), + etag=etag, + data=data, + last_modified=last_modified, + ignore_if_range=ignore_if_range, + ) + + +def remove_entity_headers( + headers: ds.Headers | list[tuple[str, str]], + allowed: t.Iterable[str] = ("expires", "content-location"), +) -> None: + """Remove all entity headers from a list or :class:`Headers` object. This + operation works in-place. `Expires` and `Content-Location` headers are + by default not removed. The reason for this is :rfc:`2616` section + 10.3.5 which specifies some entity headers that should be sent. + + .. versionchanged:: 0.5 + added `allowed` parameter. + + :param headers: a list or :class:`Headers` object. + :param allowed: a list of headers that should still be allowed even though + they are entity headers. + """ + allowed = {x.lower() for x in allowed} + headers[:] = [ + (key, value) + for key, value in headers + if not is_entity_header(key) or key.lower() in allowed + ] + + +def remove_hop_by_hop_headers(headers: ds.Headers | list[tuple[str, str]]) -> None: + """Remove all HTTP/1.1 "Hop-by-Hop" headers from a list or + :class:`Headers` object. This operation works in-place. + + .. versionadded:: 0.5 + + :param headers: a list or :class:`Headers` object. + """ + headers[:] = [ + (key, value) for key, value in headers if not is_hop_by_hop_header(key) + ] + + +def is_entity_header(header: str) -> bool: + """Check if a header is an entity header. + + .. versionadded:: 0.5 + + :param header: the header to test. + :return: `True` if it's an entity header, `False` otherwise. + """ + return header.lower() in _entity_headers + + +def is_hop_by_hop_header(header: str) -> bool: + """Check if a header is an HTTP/1.1 "Hop-by-Hop" header. + + .. versionadded:: 0.5 + + :param header: the header to test. + :return: `True` if it's an HTTP/1.1 "Hop-by-Hop" header, `False` otherwise. + """ + return header.lower() in _hop_by_hop_headers + + +def parse_cookie( + header: WSGIEnvironment | str | None, + cls: type[ds.MultiDict[str, str]] | None = None, +) -> ds.MultiDict[str, str]: + """Parse a cookie from a string or WSGI environ. + + The same key can be provided multiple times, the values are stored + in-order. The default :class:`MultiDict` will have the first value + first, and all values can be retrieved with + :meth:`MultiDict.getlist`. + + :param header: The cookie header as a string, or a WSGI environ dict + with a ``HTTP_COOKIE`` key. + :param cls: A dict-like class to store the parsed cookies in. + Defaults to :class:`MultiDict`. + + .. versionchanged:: 3.0 + Passing bytes, and the ``charset`` and ``errors`` parameters, were removed. + + .. versionchanged:: 1.0 + Returns a :class:`MultiDict` instead of a ``TypeConversionDict``. + + .. versionchanged:: 0.5 + Returns a :class:`TypeConversionDict` instead of a regular dict. The ``cls`` + parameter was added. + """ + if isinstance(header, dict): + cookie = header.get("HTTP_COOKIE") + else: + cookie = header + + if cookie: + cookie = cookie.encode("latin1").decode() + + return _sansio_http.parse_cookie(cookie=cookie, cls=cls) + + +_cookie_no_quote_re = re.compile(r"[\w!#$%&'()*+\-./:<=>?@\[\]^`{|}~]*", re.A) +_cookie_slash_re = re.compile(rb"[\x00-\x19\",;\\\x7f-\xff]", re.A) +_cookie_slash_map = {b'"': b'\\"', b"\\": b"\\\\"} +_cookie_slash_map.update( + (v.to_bytes(1, "big"), b"\\%03o" % v) + for v in [*range(0x20), *b",;", *range(0x7F, 256)] +) + + +def dump_cookie( + key: str, + value: str = "", + max_age: timedelta | int | None = None, + expires: str | datetime | int | float | None = None, + path: str | None = "/", + domain: str | None = None, + secure: bool = False, + httponly: bool = False, + sync_expires: bool = True, + max_size: int = 4093, + samesite: str | None = None, + partitioned: bool = False, +) -> str: + """Create a Set-Cookie header without the ``Set-Cookie`` prefix. + + The return value is usually restricted to ascii as the vast majority + of values are properly escaped, but that is no guarantee. It's + tunneled through latin1 as required by :pep:`3333`. + + The return value is not ASCII safe if the key contains unicode + characters. This is technically against the specification but + happens in the wild. It's strongly recommended to not use + non-ASCII values for the keys. + + :param max_age: should be a number of seconds, or `None` (default) if + the cookie should last only as long as the client's + browser session. Additionally `timedelta` objects + are accepted, too. + :param expires: should be a `datetime` object or unix timestamp. + :param path: limits the cookie to a given path, per default it will + span the whole domain. + :param domain: Use this if you want to set a cross-domain cookie. For + example, ``domain="example.com"`` will set a cookie + that is readable by the domain ``www.example.com``, + ``foo.example.com`` etc. Otherwise, a cookie will only + be readable by the domain that set it. + :param secure: The cookie will only be available via HTTPS + :param httponly: disallow JavaScript to access the cookie. This is an + extension to the cookie standard and probably not + supported by all browsers. + :param charset: the encoding for string values. + :param sync_expires: automatically set expires if max_age is defined + but expires not. + :param max_size: Warn if the final header value exceeds this size. The + default, 4093, should be safely `supported by most browsers + `_. Set to 0 to disable this check. + :param samesite: Limits the scope of the cookie such that it will + only be attached to requests if those requests are same-site. + :param partitioned: Opts the cookie into partitioned storage. This + will also set secure to True + + .. _`cookie`: http://browsercookielimits.squawky.net/ + + .. versionchanged:: 3.1 + The ``partitioned`` parameter was added. + + .. versionchanged:: 3.0 + Passing bytes, and the ``charset`` parameter, were removed. + + .. versionchanged:: 2.3.3 + The ``path`` parameter is ``/`` by default. + + .. versionchanged:: 2.3.1 + The value allows more characters without quoting. + + .. versionchanged:: 2.3 + ``localhost`` and other names without a dot are allowed for the domain. A + leading dot is ignored. + + .. versionchanged:: 2.3 + The ``path`` parameter is ``None`` by default. + + .. versionchanged:: 1.0.0 + The string ``'None'`` is accepted for ``samesite``. + """ + if path is not None: + # safe = https://url.spec.whatwg.org/#url-path-segment-string + # as well as percent for things that are already quoted + # excluding semicolon since it's part of the header syntax + path = quote(path, safe="%!$&'()*+,/:=@") + + if domain: + domain = domain.partition(":")[0].lstrip(".").encode("idna").decode("ascii") + + if isinstance(max_age, timedelta): + max_age = int(max_age.total_seconds()) + + if expires is not None: + if not isinstance(expires, str): + expires = http_date(expires) + elif max_age is not None and sync_expires: + expires = http_date(datetime.now(tz=timezone.utc).timestamp() + max_age) + + if samesite is not None: + samesite = samesite.title() + + if samesite not in {"Strict", "Lax", "None"}: + raise ValueError("SameSite must be 'Strict', 'Lax', or 'None'.") + + if partitioned: + secure = True + + # Quote value if it contains characters not allowed by RFC 6265. Slash-escape with + # three octal digits, which matches http.cookies, although the RFC suggests base64. + if not _cookie_no_quote_re.fullmatch(value): + # Work with bytes here, since a UTF-8 character could be multiple bytes. + value = _cookie_slash_re.sub( + lambda m: _cookie_slash_map[m.group()], value.encode() + ).decode("ascii") + value = f'"{value}"' + + # Send a non-ASCII key as mojibake. Everything else should already be ASCII. + # TODO Remove encoding dance, it seems like clients accept UTF-8 keys + buf = [f"{key.encode().decode('latin1')}={value}"] + + for k, v in ( + ("Domain", domain), + ("Expires", expires), + ("Max-Age", max_age), + ("Secure", secure), + ("HttpOnly", httponly), + ("Path", path), + ("SameSite", samesite), + ("Partitioned", partitioned), + ): + if v is None or v is False: + continue + + if v is True: + buf.append(k) + continue + + buf.append(f"{k}={v}") + + rv = "; ".join(buf) + + # Warn if the final value of the cookie is larger than the limit. If the cookie is + # too large, then it may be silently ignored by the browser, which can be quite hard + # to debug. + cookie_size = len(rv) + + if max_size and cookie_size > max_size: + value_size = len(value) + warnings.warn( + f"The '{key}' cookie is too large: the value was {value_size} bytes but the" + f" header required {cookie_size - value_size} extra bytes. The final size" + f" was {cookie_size} bytes but the limit is {max_size} bytes. Browsers may" + " silently ignore cookies larger than this.", + stacklevel=2, + ) + + return rv + + +def is_byte_range_valid( + start: int | None, stop: int | None, length: int | None +) -> bool: + """Checks if a given byte content range is valid for the given length. + + .. versionadded:: 0.7 + """ + if (start is None) != (stop is None): + return False + elif start is None: + return length is None or length >= 0 + elif length is None: + return 0 <= start < stop # type: ignore + elif start >= stop: # type: ignore + return False + return 0 <= start < length + + +# circular dependencies +from . import datastructures as ds # noqa: E402 +from .sansio import http as _sansio_http # noqa: E402 diff --git a/lib/python3.12/site-packages/werkzeug/local.py b/lib/python3.12/site-packages/werkzeug/local.py new file mode 100644 index 0000000000000000000000000000000000000000..32c1aca80685f6eb9745ed6d73ae3e2ca660394c --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/local.py @@ -0,0 +1,653 @@ +from __future__ import annotations + +import copy +import math +import operator +import typing as t +from contextvars import ContextVar +from functools import partial +from functools import update_wrapper +from operator import attrgetter + +from .wsgi import ClosingIterator + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + +T = t.TypeVar("T") +F = t.TypeVar("F", bound=t.Callable[..., t.Any]) + + +def release_local(local: Local | LocalStack[t.Any]) -> None: + """Release the data for the current context in a :class:`Local` or + :class:`LocalStack` without using a :class:`LocalManager`. + + This should not be needed for modern use cases, and may be removed + in the future. + + .. versionadded:: 0.6.1 + """ + local.__release_local__() + + +class Local: + """Create a namespace of context-local data. This wraps a + :class:`ContextVar` containing a :class:`dict` value. + + This may incur a performance penalty compared to using individual + context vars, as it has to copy data to avoid mutating the dict + between nested contexts. + + :param context_var: The :class:`~contextvars.ContextVar` to use as + storage for this local. If not given, one will be created. + Context vars not created at the global scope may interfere with + garbage collection. + + .. versionchanged:: 2.0 + Uses ``ContextVar`` instead of a custom storage implementation. + """ + + __slots__ = ("__storage",) + + def __init__(self, context_var: ContextVar[dict[str, t.Any]] | None = None) -> None: + if context_var is None: + # A ContextVar not created at global scope interferes with + # Python's garbage collection. However, a local only makes + # sense defined at the global scope as well, in which case + # the GC issue doesn't seem relevant. + context_var = ContextVar(f"werkzeug.Local<{id(self)}>.storage") + + object.__setattr__(self, "_Local__storage", context_var) + + def __iter__(self) -> t.Iterator[tuple[str, t.Any]]: + return iter(self.__storage.get({}).items()) + + def __call__( + self, name: str, *, unbound_message: str | None = None + ) -> LocalProxy[t.Any]: + """Create a :class:`LocalProxy` that access an attribute on this + local namespace. + + :param name: Proxy this attribute. + :param unbound_message: The error message that the proxy will + show if the attribute isn't set. + """ + return LocalProxy(self, name, unbound_message=unbound_message) + + def __release_local__(self) -> None: + self.__storage.set({}) + + def __getattr__(self, name: str) -> t.Any: + values = self.__storage.get({}) + + if name in values: + return values[name] + + raise AttributeError(name) + + def __setattr__(self, name: str, value: t.Any) -> None: + values = self.__storage.get({}).copy() + values[name] = value + self.__storage.set(values) + + def __delattr__(self, name: str) -> None: + values = self.__storage.get({}) + + if name in values: + values = values.copy() + del values[name] + self.__storage.set(values) + else: + raise AttributeError(name) + + +class LocalStack(t.Generic[T]): + """Create a stack of context-local data. This wraps a + :class:`ContextVar` containing a :class:`list` value. + + This may incur a performance penalty compared to using individual + context vars, as it has to copy data to avoid mutating the list + between nested contexts. + + :param context_var: The :class:`~contextvars.ContextVar` to use as + storage for this local. If not given, one will be created. + Context vars not created at the global scope may interfere with + garbage collection. + + .. versionchanged:: 2.0 + Uses ``ContextVar`` instead of a custom storage implementation. + + .. versionadded:: 0.6.1 + """ + + __slots__ = ("_storage",) + + def __init__(self, context_var: ContextVar[list[T]] | None = None) -> None: + if context_var is None: + # A ContextVar not created at global scope interferes with + # Python's garbage collection. However, a local only makes + # sense defined at the global scope as well, in which case + # the GC issue doesn't seem relevant. + context_var = ContextVar(f"werkzeug.LocalStack<{id(self)}>.storage") + + self._storage = context_var + + def __release_local__(self) -> None: + self._storage.set([]) + + def push(self, obj: T) -> list[T]: + """Add a new item to the top of the stack.""" + stack = self._storage.get([]).copy() + stack.append(obj) + self._storage.set(stack) + return stack + + def pop(self) -> T | None: + """Remove the top item from the stack and return it. If the + stack is empty, return ``None``. + """ + stack = self._storage.get([]) + + if len(stack) == 0: + return None + + rv = stack[-1] + self._storage.set(stack[:-1]) + return rv + + @property + def top(self) -> T | None: + """The topmost item on the stack. If the stack is empty, + `None` is returned. + """ + stack = self._storage.get([]) + + if len(stack) == 0: + return None + + return stack[-1] + + def __call__( + self, name: str | None = None, *, unbound_message: str | None = None + ) -> LocalProxy[t.Any]: + """Create a :class:`LocalProxy` that accesses the top of this + local stack. + + :param name: If given, the proxy access this attribute of the + top item, rather than the item itself. + :param unbound_message: The error message that the proxy will + show if the stack is empty. + """ + return LocalProxy(self, name, unbound_message=unbound_message) + + +class LocalManager: + """Manage releasing the data for the current context in one or more + :class:`Local` and :class:`LocalStack` objects. + + This should not be needed for modern use cases, and may be removed + in the future. + + :param locals: A local or list of locals to manage. + + .. versionchanged:: 2.1 + The ``ident_func`` was removed. + + .. versionchanged:: 0.7 + The ``ident_func`` parameter was added. + + .. versionchanged:: 0.6.1 + The :func:`release_local` function can be used instead of a + manager. + """ + + __slots__ = ("locals",) + + def __init__( + self, + locals: None + | (Local | LocalStack[t.Any] | t.Iterable[Local | LocalStack[t.Any]]) = None, + ) -> None: + if locals is None: + self.locals = [] + elif isinstance(locals, Local): + self.locals = [locals] + else: + self.locals = list(locals) # type: ignore[arg-type] + + def cleanup(self) -> None: + """Release the data in the locals for this context. Call this at + the end of each request or use :meth:`make_middleware`. + """ + for local in self.locals: + release_local(local) + + def make_middleware(self, app: WSGIApplication) -> WSGIApplication: + """Wrap a WSGI application so that local data is released + automatically after the response has been sent for a request. + """ + + def application( + environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + return ClosingIterator(app(environ, start_response), self.cleanup) + + return application + + def middleware(self, func: WSGIApplication) -> WSGIApplication: + """Like :meth:`make_middleware` but used as a decorator on the + WSGI application function. + + .. code-block:: python + + @manager.middleware + def application(environ, start_response): + ... + """ + return update_wrapper(self.make_middleware(func), func) + + def __repr__(self) -> str: + return f"<{type(self).__name__} storages: {len(self.locals)}>" + + +class _ProxyLookup: + """Descriptor that handles proxied attribute lookup for + :class:`LocalProxy`. + + :param f: The built-in function this attribute is accessed through. + Instead of looking up the special method, the function call + is redone on the object. + :param fallback: Return this function if the proxy is unbound + instead of raising a :exc:`RuntimeError`. + :param is_attr: This proxied name is an attribute, not a function. + Call the fallback immediately to get the value. + :param class_value: Value to return when accessed from the + ``LocalProxy`` class directly. Used for ``__doc__`` so building + docs still works. + """ + + __slots__ = ("bind_f", "fallback", "is_attr", "class_value", "name") + + def __init__( + self, + f: t.Callable[..., t.Any] | None = None, + fallback: t.Callable[[LocalProxy[t.Any]], t.Any] | None = None, + class_value: t.Any | None = None, + is_attr: bool = False, + ) -> None: + bind_f: t.Callable[[LocalProxy[t.Any], t.Any], t.Callable[..., t.Any]] | None + + if hasattr(f, "__get__"): + # A Python function, can be turned into a bound method. + + def bind_f( + instance: LocalProxy[t.Any], obj: t.Any + ) -> t.Callable[..., t.Any]: + return f.__get__(obj, type(obj)) # type: ignore + + elif f is not None: + # A C function, use partial to bind the first argument. + + def bind_f( + instance: LocalProxy[t.Any], obj: t.Any + ) -> t.Callable[..., t.Any]: + return partial(f, obj) + + else: + # Use getattr, which will produce a bound method. + bind_f = None + + self.bind_f = bind_f + self.fallback = fallback + self.class_value = class_value + self.is_attr = is_attr + + def __set_name__(self, owner: LocalProxy[t.Any], name: str) -> None: + self.name = name + + def __get__(self, instance: LocalProxy[t.Any], owner: type | None = None) -> t.Any: + if instance is None: + if self.class_value is not None: + return self.class_value + + return self + + try: + obj = instance._get_current_object() + except RuntimeError: + if self.fallback is None: + raise + + fallback = self.fallback.__get__(instance, owner) + + if self.is_attr: + # __class__ and __doc__ are attributes, not methods. + # Call the fallback to get the value. + return fallback() + + return fallback + + if self.bind_f is not None: + return self.bind_f(instance, obj) + + return getattr(obj, self.name) + + def __repr__(self) -> str: + return f"proxy {self.name}" + + def __call__( + self, instance: LocalProxy[t.Any], *args: t.Any, **kwargs: t.Any + ) -> t.Any: + """Support calling unbound methods from the class. For example, + this happens with ``copy.copy``, which does + ``type(x).__copy__(x)``. ``type(x)`` can't be proxied, so it + returns the proxy type and descriptor. + """ + return self.__get__(instance, type(instance))(*args, **kwargs) + + +class _ProxyIOp(_ProxyLookup): + """Look up an augmented assignment method on a proxied object. The + method is wrapped to return the proxy instead of the object. + """ + + __slots__ = () + + def __init__( + self, + f: t.Callable[..., t.Any] | None = None, + fallback: t.Callable[[LocalProxy[t.Any]], t.Any] | None = None, + ) -> None: + super().__init__(f, fallback) + + def bind_f(instance: LocalProxy[t.Any], obj: t.Any) -> t.Callable[..., t.Any]: + def i_op(self: t.Any, other: t.Any) -> LocalProxy[t.Any]: + f(self, other) # type: ignore + return instance + + return i_op.__get__(obj, type(obj)) # type: ignore + + self.bind_f = bind_f + + +def _l_to_r_op(op: F) -> F: + """Swap the argument order to turn an l-op into an r-op.""" + + def r_op(obj: t.Any, other: t.Any) -> t.Any: + return op(other, obj) + + return t.cast(F, r_op) + + +def _identity(o: T) -> T: + return o + + +class LocalProxy(t.Generic[T]): + """A proxy to the object bound to a context-local object. All + operations on the proxy are forwarded to the bound object. If no + object is bound, a ``RuntimeError`` is raised. + + :param local: The context-local object that provides the proxied + object. + :param name: Proxy this attribute from the proxied object. + :param unbound_message: The error message to show if the + context-local object is unbound. + + Proxy a :class:`~contextvars.ContextVar` to make it easier to + access. Pass a name to proxy that attribute. + + .. code-block:: python + + _request_var = ContextVar("request") + request = LocalProxy(_request_var) + session = LocalProxy(_request_var, "session") + + Proxy an attribute on a :class:`Local` namespace by calling the + local with the attribute name: + + .. code-block:: python + + data = Local() + user = data("user") + + Proxy the top item on a :class:`LocalStack` by calling the local. + Pass a name to proxy that attribute. + + .. code-block:: + + app_stack = LocalStack() + current_app = app_stack() + g = app_stack("g") + + Pass a function to proxy the return value from that function. This + was previously used to access attributes of local objects before + that was supported directly. + + .. code-block:: python + + session = LocalProxy(lambda: request.session) + + ``__repr__`` and ``__class__`` are proxied, so ``repr(x)`` and + ``isinstance(x, cls)`` will look like the proxied object. Use + ``issubclass(type(x), LocalProxy)`` to check if an object is a + proxy. + + .. code-block:: python + + repr(user) # + isinstance(user, User) # True + issubclass(type(user), LocalProxy) # True + + .. versionchanged:: 2.2.2 + ``__wrapped__`` is set when wrapping an object, not only when + wrapping a function, to prevent doctest from failing. + + .. versionchanged:: 2.2 + Can proxy a ``ContextVar`` or ``LocalStack`` directly. + + .. versionchanged:: 2.2 + The ``name`` parameter can be used with any proxied object, not + only ``Local``. + + .. versionchanged:: 2.2 + Added the ``unbound_message`` parameter. + + .. versionchanged:: 2.0 + Updated proxied attributes and methods to reflect the current + data model. + + .. versionchanged:: 0.6.1 + The class can be instantiated with a callable. + """ + + __slots__ = ("__wrapped", "_get_current_object") + + _get_current_object: t.Callable[[], T] + """Return the current object this proxy is bound to. If the proxy is + unbound, this raises a ``RuntimeError``. + + This should be used if you need to pass the object to something that + doesn't understand the proxy. It can also be useful for performance + if you are accessing the object multiple times in a function, rather + than going through the proxy multiple times. + """ + + def __init__( + self, + local: ContextVar[T] | Local | LocalStack[T] | t.Callable[[], T], + name: str | None = None, + *, + unbound_message: str | None = None, + ) -> None: + if name is None: + get_name = _identity + else: + get_name = attrgetter(name) # type: ignore[assignment] + + if unbound_message is None: + unbound_message = "object is not bound" + + if isinstance(local, Local): + if name is None: + raise TypeError("'name' is required when proxying a 'Local' object.") + + def _get_current_object() -> T: + try: + return get_name(local) # type: ignore[return-value] + except AttributeError: + raise RuntimeError(unbound_message) from None + + elif isinstance(local, LocalStack): + + def _get_current_object() -> T: + obj = local.top + + if obj is None: + raise RuntimeError(unbound_message) + + return get_name(obj) + + elif isinstance(local, ContextVar): + + def _get_current_object() -> T: + try: + obj = local.get() + except LookupError: + raise RuntimeError(unbound_message) from None + + return get_name(obj) + + elif callable(local): + + def _get_current_object() -> T: + return get_name(local()) + + else: + raise TypeError(f"Don't know how to proxy '{type(local)}'.") + + object.__setattr__(self, "_LocalProxy__wrapped", local) + object.__setattr__(self, "_get_current_object", _get_current_object) + + __doc__ = _ProxyLookup( + class_value=__doc__, fallback=lambda self: type(self).__doc__, is_attr=True + ) + __wrapped__ = _ProxyLookup( + fallback=lambda self: self._LocalProxy__wrapped, # type: ignore[attr-defined] + is_attr=True, + ) + # __del__ should only delete the proxy + __repr__ = _ProxyLookup( + repr, fallback=lambda self: f"<{type(self).__name__} unbound>" + ) + __str__ = _ProxyLookup(str) + __bytes__ = _ProxyLookup(bytes) + __format__ = _ProxyLookup() + __lt__ = _ProxyLookup(operator.lt) + __le__ = _ProxyLookup(operator.le) + __eq__ = _ProxyLookup(operator.eq) + __ne__ = _ProxyLookup(operator.ne) + __gt__ = _ProxyLookup(operator.gt) + __ge__ = _ProxyLookup(operator.ge) + __hash__ = _ProxyLookup(hash) + __bool__ = _ProxyLookup(bool, fallback=lambda self: False) + __getattr__ = _ProxyLookup(getattr) + # __getattribute__ triggered through __getattr__ + __setattr__ = _ProxyLookup(setattr) + __delattr__ = _ProxyLookup(delattr) + __dir__ = _ProxyLookup(dir, fallback=lambda self: []) + # __get__ (proxying descriptor not supported) + # __set__ (descriptor) + # __delete__ (descriptor) + # __set_name__ (descriptor) + # __objclass__ (descriptor) + # __slots__ used by proxy itself + # __dict__ (__getattr__) + # __weakref__ (__getattr__) + # __init_subclass__ (proxying metaclass not supported) + # __prepare__ (metaclass) + __class__ = _ProxyLookup(fallback=lambda self: type(self), is_attr=True) + __instancecheck__ = _ProxyLookup(lambda self, other: isinstance(other, self)) + __subclasscheck__ = _ProxyLookup(lambda self, other: issubclass(other, self)) + # __class_getitem__ triggered through __getitem__ + __call__ = _ProxyLookup(lambda self, *args, **kwargs: self(*args, **kwargs)) + __len__ = _ProxyLookup(len) + __length_hint__ = _ProxyLookup(operator.length_hint) + __getitem__ = _ProxyLookup(operator.getitem) + __setitem__ = _ProxyLookup(operator.setitem) + __delitem__ = _ProxyLookup(operator.delitem) + # __missing__ triggered through __getitem__ + __iter__ = _ProxyLookup(iter) + __next__ = _ProxyLookup(next) + __reversed__ = _ProxyLookup(reversed) + __contains__ = _ProxyLookup(operator.contains) + __add__ = _ProxyLookup(operator.add) + __sub__ = _ProxyLookup(operator.sub) + __mul__ = _ProxyLookup(operator.mul) + __matmul__ = _ProxyLookup(operator.matmul) + __truediv__ = _ProxyLookup(operator.truediv) + __floordiv__ = _ProxyLookup(operator.floordiv) + __mod__ = _ProxyLookup(operator.mod) + __divmod__ = _ProxyLookup(divmod) + __pow__ = _ProxyLookup(pow) + __lshift__ = _ProxyLookup(operator.lshift) + __rshift__ = _ProxyLookup(operator.rshift) + __and__ = _ProxyLookup(operator.and_) + __xor__ = _ProxyLookup(operator.xor) + __or__ = _ProxyLookup(operator.or_) + __radd__ = _ProxyLookup(_l_to_r_op(operator.add)) + __rsub__ = _ProxyLookup(_l_to_r_op(operator.sub)) + __rmul__ = _ProxyLookup(_l_to_r_op(operator.mul)) + __rmatmul__ = _ProxyLookup(_l_to_r_op(operator.matmul)) + __rtruediv__ = _ProxyLookup(_l_to_r_op(operator.truediv)) + __rfloordiv__ = _ProxyLookup(_l_to_r_op(operator.floordiv)) + __rmod__ = _ProxyLookup(_l_to_r_op(operator.mod)) + __rdivmod__ = _ProxyLookup(_l_to_r_op(divmod)) + __rpow__ = _ProxyLookup(_l_to_r_op(pow)) + __rlshift__ = _ProxyLookup(_l_to_r_op(operator.lshift)) + __rrshift__ = _ProxyLookup(_l_to_r_op(operator.rshift)) + __rand__ = _ProxyLookup(_l_to_r_op(operator.and_)) + __rxor__ = _ProxyLookup(_l_to_r_op(operator.xor)) + __ror__ = _ProxyLookup(_l_to_r_op(operator.or_)) + __iadd__ = _ProxyIOp(operator.iadd) + __isub__ = _ProxyIOp(operator.isub) + __imul__ = _ProxyIOp(operator.imul) + __imatmul__ = _ProxyIOp(operator.imatmul) + __itruediv__ = _ProxyIOp(operator.itruediv) + __ifloordiv__ = _ProxyIOp(operator.ifloordiv) + __imod__ = _ProxyIOp(operator.imod) + __ipow__ = _ProxyIOp(operator.ipow) + __ilshift__ = _ProxyIOp(operator.ilshift) + __irshift__ = _ProxyIOp(operator.irshift) + __iand__ = _ProxyIOp(operator.iand) + __ixor__ = _ProxyIOp(operator.ixor) + __ior__ = _ProxyIOp(operator.ior) + __neg__ = _ProxyLookup(operator.neg) + __pos__ = _ProxyLookup(operator.pos) + __abs__ = _ProxyLookup(abs) + __invert__ = _ProxyLookup(operator.invert) + __complex__ = _ProxyLookup(complex) + __int__ = _ProxyLookup(int) + __float__ = _ProxyLookup(float) + __index__ = _ProxyLookup(operator.index) + __round__ = _ProxyLookup(round) + __trunc__ = _ProxyLookup(math.trunc) + __floor__ = _ProxyLookup(math.floor) + __ceil__ = _ProxyLookup(math.ceil) + __enter__ = _ProxyLookup() + __exit__ = _ProxyLookup() + __await__ = _ProxyLookup() + __aiter__ = _ProxyLookup() + __anext__ = _ProxyLookup() + __aenter__ = _ProxyLookup() + __aexit__ = _ProxyLookup() + __copy__ = _ProxyLookup(copy.copy) + __deepcopy__ = _ProxyLookup(copy.deepcopy) + # __getnewargs_ex__ (pickle through proxy not supported) + # __getnewargs__ (pickle) + # __getstate__ (pickle) + # __setstate__ (pickle) + # __reduce__ (pickle) + # __reduce_ex__ (pickle) diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__init__.py b/lib/python3.12/site-packages/werkzeug/middleware/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6363e2bdae80b0719c5c5055f5b4dcff643b9281 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/dispatcher.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/dispatcher.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc03100ffa0c08cf06ccf0298782476ac4fc1fba Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/dispatcher.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/http_proxy.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/http_proxy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e11fd0b944e4cae7a1070431179206e43f7a84c2 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/http_proxy.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/lint.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/lint.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2472ac8d008c6c77feb300a6101a71ea47ac46b4 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/lint.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/profiler.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/profiler.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ba619702a078a17dbec4e8ac5a1848542ab74e4 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/profiler.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/proxy_fix.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/proxy_fix.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..218989686a1c594fca2b01745ef4759da95c01b8 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/proxy_fix.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/shared_data.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/shared_data.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d6b466ebfaf257776c77a38b5f22efa942c94514 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/middleware/__pycache__/shared_data.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/middleware/dispatcher.py b/lib/python3.12/site-packages/werkzeug/middleware/dispatcher.py new file mode 100644 index 0000000000000000000000000000000000000000..e11bacc527b2c2ce82ff62ea0d4a9c3ef5514b12 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/middleware/dispatcher.py @@ -0,0 +1,81 @@ +""" +Application Dispatcher +====================== + +This middleware creates a single WSGI application that dispatches to +multiple other WSGI applications mounted at different URL paths. + +A common example is writing a Single Page Application, where you have a +backend API and a frontend written in JavaScript that does the routing +in the browser rather than requesting different pages from the server. +The frontend is a single HTML and JS file that should be served for any +path besides "/api". + +This example dispatches to an API app under "/api", an admin app +under "/admin", and an app that serves frontend files for all other +requests:: + + app = DispatcherMiddleware(serve_frontend, { + '/api': api_app, + '/admin': admin_app, + }) + +In production, you might instead handle this at the HTTP server level, +serving files or proxying to application servers based on location. The +API and admin apps would each be deployed with a separate WSGI server, +and the static files would be served directly by the HTTP server. + +.. autoclass:: DispatcherMiddleware + +:copyright: 2007 Pallets +:license: BSD-3-Clause +""" + +from __future__ import annotations + +import typing as t + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +class DispatcherMiddleware: + """Combine multiple applications as a single WSGI application. + Requests are dispatched to an application based on the path it is + mounted under. + + :param app: The WSGI application to dispatch to if the request + doesn't match a mounted path. + :param mounts: Maps path prefixes to applications for dispatching. + """ + + def __init__( + self, + app: WSGIApplication, + mounts: dict[str, WSGIApplication] | None = None, + ) -> None: + self.app = app + self.mounts = mounts or {} + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + script = environ.get("PATH_INFO", "") + path_info = "" + + while "/" in script: + if script in self.mounts: + app = self.mounts[script] + break + + script, last_item = script.rsplit("/", 1) + path_info = f"/{last_item}{path_info}" + else: + app = self.mounts.get(script, self.app) + + original_script_name = environ.get("SCRIPT_NAME", "") + environ["SCRIPT_NAME"] = original_script_name + script + environ["PATH_INFO"] = path_info + return app(environ, start_response) diff --git a/lib/python3.12/site-packages/werkzeug/middleware/http_proxy.py b/lib/python3.12/site-packages/werkzeug/middleware/http_proxy.py new file mode 100644 index 0000000000000000000000000000000000000000..5e239156aa1aef21a9ca762281ecb1185d70ed20 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/middleware/http_proxy.py @@ -0,0 +1,236 @@ +""" +Basic HTTP Proxy +================ + +.. autoclass:: ProxyMiddleware + +:copyright: 2007 Pallets +:license: BSD-3-Clause +""" + +from __future__ import annotations + +import typing as t +from http import client +from urllib.parse import quote +from urllib.parse import urlsplit + +from ..datastructures import EnvironHeaders +from ..http import is_hop_by_hop_header +from ..wsgi import get_input_stream + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +class ProxyMiddleware: + """Proxy requests under a path to an external server, routing other + requests to the app. + + This middleware can only proxy HTTP requests, as HTTP is the only + protocol handled by the WSGI server. Other protocols, such as + WebSocket requests, cannot be proxied at this layer. This should + only be used for development, in production a real proxy server + should be used. + + The middleware takes a dict mapping a path prefix to a dict + describing the host to be proxied to:: + + app = ProxyMiddleware(app, { + "/static/": { + "target": "http://127.0.0.1:5001/", + } + }) + + Each host has the following options: + + ``target``: + The target URL to dispatch to. This is required. + ``remove_prefix``: + Whether to remove the prefix from the URL before dispatching it + to the target. The default is ``False``. + ``host``: + ``""`` (default): + The host header is automatically rewritten to the URL of the + target. + ``None``: + The host header is unmodified from the client request. + Any other value: + The host header is overwritten with the value. + ``headers``: + A dictionary of headers to be sent with the request to the + target. The default is ``{}``. + ``ssl_context``: + A :class:`ssl.SSLContext` defining how to verify requests if the + target is HTTPS. The default is ``None``. + + In the example above, everything under ``"/static/"`` is proxied to + the server on port 5001. The host header is rewritten to the target, + and the ``"/static/"`` prefix is removed from the URLs. + + :param app: The WSGI application to wrap. + :param targets: Proxy target configurations. See description above. + :param chunk_size: Size of chunks to read from input stream and + write to target. + :param timeout: Seconds before an operation to a target fails. + + .. versionadded:: 0.14 + """ + + def __init__( + self, + app: WSGIApplication, + targets: t.Mapping[str, dict[str, t.Any]], + chunk_size: int = 2 << 13, + timeout: int = 10, + ) -> None: + def _set_defaults(opts: dict[str, t.Any]) -> dict[str, t.Any]: + opts.setdefault("remove_prefix", False) + opts.setdefault("host", "") + opts.setdefault("headers", {}) + opts.setdefault("ssl_context", None) + return opts + + self.app = app + self.targets = { + f"/{k.strip('/')}/": _set_defaults(v) for k, v in targets.items() + } + self.chunk_size = chunk_size + self.timeout = timeout + + def proxy_to( + self, opts: dict[str, t.Any], path: str, prefix: str + ) -> WSGIApplication: + target = urlsplit(opts["target"]) + # socket can handle unicode host, but header must be ascii + host = target.hostname.encode("idna").decode("ascii") + + def application( + environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + headers = list(EnvironHeaders(environ).items()) + headers[:] = [ + (k, v) + for k, v in headers + if not is_hop_by_hop_header(k) + and k.lower() not in ("content-length", "host") + ] + headers.append(("Connection", "close")) + + if opts["host"] == "": + headers.append(("Host", host)) + elif opts["host"] is None: + headers.append(("Host", environ["HTTP_HOST"])) + else: + headers.append(("Host", opts["host"])) + + headers.extend(opts["headers"].items()) + remote_path = path + + if opts["remove_prefix"]: + remote_path = remote_path[len(prefix) :].lstrip("/") + remote_path = f"{target.path.rstrip('/')}/{remote_path}" + + content_length = environ.get("CONTENT_LENGTH") + chunked = False + + if content_length not in ("", None): + headers.append(("Content-Length", content_length)) # type: ignore + elif content_length is not None: + headers.append(("Transfer-Encoding", "chunked")) + chunked = True + + try: + if target.scheme == "http": + con = client.HTTPConnection( + host, target.port or 80, timeout=self.timeout + ) + elif target.scheme == "https": + con = client.HTTPSConnection( + host, + target.port or 443, + timeout=self.timeout, + context=opts["ssl_context"], + ) + else: + raise RuntimeError( + "Target scheme must be 'http' or 'https', got" + f" {target.scheme!r}." + ) + + con.connect() + # safe = https://url.spec.whatwg.org/#url-path-segment-string + # as well as percent for things that are already quoted + remote_url = quote(remote_path, safe="!$&'()*+,/:;=@%") + querystring = environ["QUERY_STRING"] + + if querystring: + remote_url = f"{remote_url}?{querystring}" + + con.putrequest(environ["REQUEST_METHOD"], remote_url, skip_host=True) + + for k, v in headers: + if k.lower() == "connection": + v = "close" + + con.putheader(k, v) + + con.endheaders() + stream = get_input_stream(environ) + + while True: + data = stream.read(self.chunk_size) + + if not data: + break + + if chunked: + con.send(b"%x\r\n%s\r\n" % (len(data), data)) + else: + con.send(data) + + resp = con.getresponse() + except OSError: + from ..exceptions import BadGateway + + return BadGateway()(environ, start_response) + + start_response( + f"{resp.status} {resp.reason}", + [ + (k.title(), v) + for k, v in resp.getheaders() + if not is_hop_by_hop_header(k) + ], + ) + + def read() -> t.Iterator[bytes]: + while True: + try: + data = resp.read(self.chunk_size) + except OSError: + break + + if not data: + break + + yield data + + return read() + + return application + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + path = environ["PATH_INFO"] + app = self.app + + for prefix, opts in self.targets.items(): + if path.startswith(prefix): + app = self.proxy_to(opts, path, prefix) + break + + return app(environ, start_response) diff --git a/lib/python3.12/site-packages/werkzeug/middleware/lint.py b/lib/python3.12/site-packages/werkzeug/middleware/lint.py new file mode 100644 index 0000000000000000000000000000000000000000..3714271b1cb8894be598eb165f93d9315f130422 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/middleware/lint.py @@ -0,0 +1,439 @@ +""" +WSGI Protocol Linter +==================== + +This module provides a middleware that performs sanity checks on the +behavior of the WSGI server and application. It checks that the +:pep:`3333` WSGI spec is properly implemented. It also warns on some +common HTTP errors such as non-empty responses for 304 status codes. + +.. autoclass:: LintMiddleware + +:copyright: 2007 Pallets +:license: BSD-3-Clause +""" + +from __future__ import annotations + +import typing as t +from types import TracebackType +from urllib.parse import urlparse +from warnings import warn + +from ..datastructures import Headers +from ..http import is_entity_header +from ..wsgi import FileWrapper + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +class WSGIWarning(Warning): + """Warning class for WSGI warnings.""" + + +class HTTPWarning(Warning): + """Warning class for HTTP warnings.""" + + +def check_type(context: str, obj: object, need: type = str) -> None: + if type(obj) is not need: + warn( + f"{context!r} requires {need.__name__!r}, got {type(obj).__name__!r}.", + WSGIWarning, + stacklevel=3, + ) + + +class InputStream: + def __init__(self, stream: t.IO[bytes]) -> None: + self._stream = stream + + def read(self, *args: t.Any) -> bytes: + if len(args) == 0: + warn( + "WSGI does not guarantee an EOF marker on the input stream, thus making" + " calls to 'wsgi.input.read()' unsafe. Conforming servers may never" + " return from this call.", + WSGIWarning, + stacklevel=2, + ) + elif len(args) != 1: + warn( + "Too many parameters passed to 'wsgi.input.read()'.", + WSGIWarning, + stacklevel=2, + ) + return self._stream.read(*args) + + def readline(self, *args: t.Any) -> bytes: + if len(args) == 0: + warn( + "Calls to 'wsgi.input.readline()' without arguments are unsafe. Use" + " 'wsgi.input.read()' instead.", + WSGIWarning, + stacklevel=2, + ) + elif len(args) == 1: + warn( + "'wsgi.input.readline()' was called with a size hint. WSGI does not" + " support this, although it's available on all major servers.", + WSGIWarning, + stacklevel=2, + ) + else: + raise TypeError("Too many arguments passed to 'wsgi.input.readline()'.") + return self._stream.readline(*args) + + def __iter__(self) -> t.Iterator[bytes]: + try: + return iter(self._stream) + except TypeError: + warn("'wsgi.input' is not iterable.", WSGIWarning, stacklevel=2) + return iter(()) + + def close(self) -> None: + warn("The application closed the input stream!", WSGIWarning, stacklevel=2) + self._stream.close() + + +class ErrorStream: + def __init__(self, stream: t.IO[str]) -> None: + self._stream = stream + + def write(self, s: str) -> None: + check_type("wsgi.error.write()", s, str) + self._stream.write(s) + + def flush(self) -> None: + self._stream.flush() + + def writelines(self, seq: t.Iterable[str]) -> None: + for line in seq: + self.write(line) + + def close(self) -> None: + warn("The application closed the error stream!", WSGIWarning, stacklevel=2) + self._stream.close() + + +class GuardedWrite: + def __init__(self, write: t.Callable[[bytes], object], chunks: list[int]) -> None: + self._write = write + self._chunks = chunks + + def __call__(self, s: bytes) -> None: + check_type("write()", s, bytes) + self._write(s) + self._chunks.append(len(s)) + + +class GuardedIterator: + def __init__( + self, + iterator: t.Iterable[bytes], + headers_set: tuple[int, Headers], + chunks: list[int], + ) -> None: + self._iterator = iterator + self._next = iter(iterator).__next__ + self.closed = False + self.headers_set = headers_set + self.chunks = chunks + + def __iter__(self) -> GuardedIterator: + return self + + def __next__(self) -> bytes: + if self.closed: + warn("Iterated over closed 'app_iter'.", WSGIWarning, stacklevel=2) + + rv = self._next() + + if not self.headers_set: + warn( + "The application returned before it started the response.", + WSGIWarning, + stacklevel=2, + ) + + check_type("application iterator items", rv, bytes) + self.chunks.append(len(rv)) + return rv + + def close(self) -> None: + self.closed = True + + if hasattr(self._iterator, "close"): + self._iterator.close() + + if self.headers_set: + status_code, headers = self.headers_set + bytes_sent = sum(self.chunks) + content_length = headers.get("content-length", type=int) + + if status_code == 304: + for key, _value in headers: + key = key.lower() + if key not in ("expires", "content-location") and is_entity_header( + key + ): + warn( + f"Entity header {key!r} found in 304 response.", + HTTPWarning, + stacklevel=2, + ) + if bytes_sent: + warn( + "304 responses must not have a body.", + HTTPWarning, + stacklevel=2, + ) + elif 100 <= status_code < 200 or status_code == 204: + if content_length != 0: + warn( + f"{status_code} responses must have an empty content length.", + HTTPWarning, + stacklevel=2, + ) + if bytes_sent: + warn( + f"{status_code} responses must not have a body.", + HTTPWarning, + stacklevel=2, + ) + elif content_length is not None and content_length != bytes_sent: + warn( + "Content-Length and the number of bytes sent to the" + " client do not match.", + WSGIWarning, + stacklevel=2, + ) + + def __del__(self) -> None: + if not self.closed: + try: + warn( + "Iterator was garbage collected before it was closed.", + WSGIWarning, + stacklevel=2, + ) + except Exception: + pass + + +class LintMiddleware: + """Warns about common errors in the WSGI and HTTP behavior of the + server and wrapped application. Some of the issues it checks are: + + - invalid status codes + - non-bytes sent to the WSGI server + - strings returned from the WSGI application + - non-empty conditional responses + - unquoted etags + - relative URLs in the Location header + - unsafe calls to wsgi.input + - unclosed iterators + + Error information is emitted using the :mod:`warnings` module. + + :param app: The WSGI application to wrap. + + .. code-block:: python + + from werkzeug.middleware.lint import LintMiddleware + app = LintMiddleware(app) + """ + + def __init__(self, app: WSGIApplication) -> None: + self.app = app + + def check_environ(self, environ: WSGIEnvironment) -> None: + if type(environ) is not dict: # noqa: E721 + warn( + "WSGI environment is not a standard Python dict.", + WSGIWarning, + stacklevel=4, + ) + for key in ( + "REQUEST_METHOD", + "SERVER_NAME", + "SERVER_PORT", + "wsgi.version", + "wsgi.input", + "wsgi.errors", + "wsgi.multithread", + "wsgi.multiprocess", + "wsgi.run_once", + ): + if key not in environ: + warn( + f"Required environment key {key!r} not found", + WSGIWarning, + stacklevel=3, + ) + if environ["wsgi.version"] != (1, 0): + warn("Environ is not a WSGI 1.0 environ.", WSGIWarning, stacklevel=3) + + script_name = environ.get("SCRIPT_NAME", "") + path_info = environ.get("PATH_INFO", "") + + if script_name and script_name[0] != "/": + warn( + f"'SCRIPT_NAME' does not start with a slash: {script_name!r}", + WSGIWarning, + stacklevel=3, + ) + + if path_info and path_info[0] != "/": + warn( + f"'PATH_INFO' does not start with a slash: {path_info!r}", + WSGIWarning, + stacklevel=3, + ) + + def check_start_response( + self, + status: str, + headers: list[tuple[str, str]], + exc_info: None | (tuple[type[BaseException], BaseException, TracebackType]), + ) -> tuple[int, Headers]: + check_type("status", status, str) + status_code_str = status.split(None, 1)[0] + + if len(status_code_str) != 3 or not status_code_str.isdecimal(): + warn("Status code must be three digits.", WSGIWarning, stacklevel=3) + + if len(status) < 4 or status[3] != " ": + warn( + f"Invalid value for status {status!r}. Valid status strings are three" + " digits, a space and a status explanation.", + WSGIWarning, + stacklevel=3, + ) + + status_code = int(status_code_str) + + if status_code < 100: + warn("Status code < 100 detected.", WSGIWarning, stacklevel=3) + + if type(headers) is not list: # noqa: E721 + warn("Header list is not a list.", WSGIWarning, stacklevel=3) + + for item in headers: + if type(item) is not tuple or len(item) != 2: + warn("Header items must be 2-item tuples.", WSGIWarning, stacklevel=3) + name, value = item + if type(name) is not str or type(value) is not str: # noqa: E721 + warn( + "Header keys and values must be strings.", WSGIWarning, stacklevel=3 + ) + if name.lower() == "status": + warn( + "The status header is not supported due to" + " conflicts with the CGI spec.", + WSGIWarning, + stacklevel=3, + ) + + if exc_info is not None and not isinstance(exc_info, tuple): + warn("Invalid value for exc_info.", WSGIWarning, stacklevel=3) + + headers_obj = Headers(headers) + self.check_headers(headers_obj) + + return status_code, headers_obj + + def check_headers(self, headers: Headers) -> None: + etag = headers.get("etag") + + if etag is not None: + if etag.startswith(("W/", "w/")): + if etag.startswith("w/"): + warn( + "Weak etag indicator should be upper case.", + HTTPWarning, + stacklevel=4, + ) + + etag = etag[2:] + + if not (etag[:1] == etag[-1:] == '"'): + warn("Unquoted etag emitted.", HTTPWarning, stacklevel=4) + + location = headers.get("location") + + if location is not None: + if not urlparse(location).netloc: + warn( + "Absolute URLs required for location header.", + HTTPWarning, + stacklevel=4, + ) + + def check_iterator(self, app_iter: t.Iterable[bytes]) -> None: + if isinstance(app_iter, str): + warn( + "The application returned a string. The response will send one" + " character at a time to the client, which will kill performance." + " Return a list or iterable instead.", + WSGIWarning, + stacklevel=3, + ) + + def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Iterable[bytes]: + if len(args) != 2: + warn("A WSGI app takes two arguments.", WSGIWarning, stacklevel=2) + + if kwargs: + warn( + "A WSGI app does not take keyword arguments.", WSGIWarning, stacklevel=2 + ) + + environ: WSGIEnvironment = args[0] + start_response: StartResponse = args[1] + + self.check_environ(environ) + environ["wsgi.input"] = InputStream(environ["wsgi.input"]) + environ["wsgi.errors"] = ErrorStream(environ["wsgi.errors"]) + + # Hook our own file wrapper in so that applications will always + # iterate to the end and we can check the content length. + environ["wsgi.file_wrapper"] = FileWrapper + + headers_set: list[t.Any] = [] + chunks: list[int] = [] + + def checking_start_response( + *args: t.Any, **kwargs: t.Any + ) -> t.Callable[[bytes], None]: + if len(args) not in {2, 3}: + warn( + f"Invalid number of arguments: {len(args)}, expected 2 or 3.", + WSGIWarning, + stacklevel=2, + ) + + if kwargs: + warn( + "'start_response' does not take keyword arguments.", + WSGIWarning, + stacklevel=2, + ) + + status: str = args[0] + headers: list[tuple[str, str]] = args[1] + exc_info: ( + None | (tuple[type[BaseException], BaseException, TracebackType]) + ) = args[2] if len(args) == 3 else None + + headers_set[:] = self.check_start_response(status, headers, exc_info) + return GuardedWrite(start_response(status, headers, exc_info), chunks) + + app_iter = self.app(environ, t.cast("StartResponse", checking_start_response)) + self.check_iterator(app_iter) + return GuardedIterator( + app_iter, t.cast(tuple[int, Headers], headers_set), chunks + ) diff --git a/lib/python3.12/site-packages/werkzeug/middleware/profiler.py b/lib/python3.12/site-packages/werkzeug/middleware/profiler.py new file mode 100644 index 0000000000000000000000000000000000000000..112b877761c00ab34fd95dbb8ba938b91eb34c79 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/middleware/profiler.py @@ -0,0 +1,155 @@ +""" +Application Profiler +==================== + +This module provides a middleware that profiles each request with the +:mod:`cProfile` module. This can help identify bottlenecks in your code +that may be slowing down your application. + +.. autoclass:: ProfilerMiddleware + +:copyright: 2007 Pallets +:license: BSD-3-Clause +""" + +from __future__ import annotations + +import os.path +import sys +import time +import typing as t +from pstats import Stats + +try: + from cProfile import Profile +except ImportError: + from profile import Profile # type: ignore + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +class ProfilerMiddleware: + """Wrap a WSGI application and profile the execution of each + request. Responses are buffered so that timings are more exact. + + If ``stream`` is given, :class:`pstats.Stats` are written to it + after each request. If ``profile_dir`` is given, :mod:`cProfile` + data files are saved to that directory, one file per request. + + The filename can be customized by passing ``filename_format``. If + it is a string, it will be formatted using :meth:`str.format` with + the following fields available: + + - ``{method}`` - The request method; GET, POST, etc. + - ``{path}`` - The request path or 'root' should one not exist. + - ``{elapsed}`` - The elapsed time of the request in milliseconds. + - ``{time}`` - The time of the request. + + If it is a callable, it will be called with the WSGI ``environ`` and + be expected to return a filename string. The ``environ`` dictionary + will also have the ``"werkzeug.profiler"`` key populated with a + dictionary containing the following fields (more may be added in the + future): + - ``{elapsed}`` - The elapsed time of the request in milliseconds. + - ``{time}`` - The time of the request. + + :param app: The WSGI application to wrap. + :param stream: Write stats to this stream. Disable with ``None``. + :param sort_by: A tuple of columns to sort stats by. See + :meth:`pstats.Stats.sort_stats`. + :param restrictions: A tuple of restrictions to filter stats by. See + :meth:`pstats.Stats.print_stats`. + :param profile_dir: Save profile data files to this directory. + :param filename_format: Format string for profile data file names, + or a callable returning a name. See explanation above. + + .. code-block:: python + + from werkzeug.middleware.profiler import ProfilerMiddleware + app = ProfilerMiddleware(app) + + .. versionchanged:: 3.0 + Added the ``"werkzeug.profiler"`` key to the ``filename_format(environ)`` + parameter with the ``elapsed`` and ``time`` fields. + + .. versionchanged:: 0.15 + Stats are written even if ``profile_dir`` is given, and can be + disable by passing ``stream=None``. + + .. versionadded:: 0.15 + Added ``filename_format``. + + .. versionadded:: 0.9 + Added ``restrictions`` and ``profile_dir``. + """ + + def __init__( + self, + app: WSGIApplication, + stream: t.IO[str] | None = sys.stdout, + sort_by: t.Iterable[str] = ("time", "calls"), + restrictions: t.Iterable[str | int | float] = (), + profile_dir: str | None = None, + filename_format: str = "{method}.{path}.{elapsed:.0f}ms.{time:.0f}.prof", + ) -> None: + self._app = app + self._stream = stream + self._sort_by = sort_by + self._restrictions = restrictions + self._profile_dir = profile_dir + self._filename_format = filename_format + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + response_body: list[bytes] = [] + + def catching_start_response(status, headers, exc_info=None): # type: ignore + start_response(status, headers, exc_info) + return response_body.append + + def runapp() -> None: + app_iter = self._app( + environ, t.cast("StartResponse", catching_start_response) + ) + response_body.extend(app_iter) + + if hasattr(app_iter, "close"): + app_iter.close() + + profile = Profile() + start = time.time() + profile.runcall(runapp) + body = b"".join(response_body) + elapsed = time.time() - start + + if self._profile_dir is not None: + if callable(self._filename_format): + environ["werkzeug.profiler"] = { + "elapsed": elapsed * 1000.0, + "time": time.time(), + } + filename = self._filename_format(environ) + else: + filename = self._filename_format.format( + method=environ["REQUEST_METHOD"], + path=environ["PATH_INFO"].strip("/").replace("/", ".") or "root", + elapsed=elapsed * 1000.0, + time=time.time(), + ) + filename = os.path.join(self._profile_dir, filename) + profile.dump_stats(filename) + + if self._stream is not None: + stats = Stats(profile, stream=self._stream) + stats.sort_stats(*self._sort_by) + print("-" * 80, file=self._stream) + path_info = environ.get("PATH_INFO", "") + print(f"PATH: {path_info!r}", file=self._stream) + stats.print_stats(*self._restrictions) + print(f"{'-' * 80}\n", file=self._stream) + + return [body] diff --git a/lib/python3.12/site-packages/werkzeug/middleware/proxy_fix.py b/lib/python3.12/site-packages/werkzeug/middleware/proxy_fix.py new file mode 100644 index 0000000000000000000000000000000000000000..cbf4e0bae0c1077f0f3db8e776efd655ab799596 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/middleware/proxy_fix.py @@ -0,0 +1,183 @@ +""" +X-Forwarded-For Proxy Fix +========================= + +This module provides a middleware that adjusts the WSGI environ based on +``X-Forwarded-`` headers that proxies in front of an application may +set. + +When an application is running behind a proxy server, WSGI may see the +request as coming from that server rather than the real client. Proxies +set various headers to track where the request actually came from. + +This middleware should only be used if the application is actually +behind such a proxy, and should be configured with the number of proxies +that are chained in front of it. Not all proxies set all the headers. +Since incoming headers can be faked, you must set how many proxies are +setting each header so the middleware knows what to trust. + +.. autoclass:: ProxyFix + +:copyright: 2007 Pallets +:license: BSD-3-Clause +""" + +from __future__ import annotations + +import typing as t + +from ..http import parse_list_header + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +class ProxyFix: + """Adjust the WSGI environ based on ``X-Forwarded-`` that proxies in + front of the application may set. + + - ``X-Forwarded-For`` sets ``REMOTE_ADDR``. + - ``X-Forwarded-Proto`` sets ``wsgi.url_scheme``. + - ``X-Forwarded-Host`` sets ``HTTP_HOST``, ``SERVER_NAME``, and + ``SERVER_PORT``. + - ``X-Forwarded-Port`` sets ``HTTP_HOST`` and ``SERVER_PORT``. + - ``X-Forwarded-Prefix`` sets ``SCRIPT_NAME``. + + You must tell the middleware how many proxies set each header so it + knows what values to trust. It is a security issue to trust values + that came from the client rather than a proxy. + + The original values of the headers are stored in the WSGI + environ as ``werkzeug.proxy_fix.orig``, a dict. + + :param app: The WSGI application to wrap. + :param x_for: Number of values to trust for ``X-Forwarded-For``. + :param x_proto: Number of values to trust for ``X-Forwarded-Proto``. + :param x_host: Number of values to trust for ``X-Forwarded-Host``. + :param x_port: Number of values to trust for ``X-Forwarded-Port``. + :param x_prefix: Number of values to trust for + ``X-Forwarded-Prefix``. + + .. code-block:: python + + from werkzeug.middleware.proxy_fix import ProxyFix + # App is behind one proxy that sets the -For and -Host headers. + app = ProxyFix(app, x_for=1, x_host=1) + + .. versionchanged:: 1.0 + The ``num_proxies`` argument and attribute; the ``get_remote_addr`` method; and + the environ keys ``orig_remote_addr``, ``orig_wsgi_url_scheme``, and + ``orig_http_host`` were removed. + + .. versionchanged:: 0.15 + All headers support multiple values. Each header is configured with a separate + number of trusted proxies. + + .. versionchanged:: 0.15 + Original WSGI environ values are stored in the ``werkzeug.proxy_fix.orig`` dict. + + .. versionchanged:: 0.15 + Support ``X-Forwarded-Port`` and ``X-Forwarded-Prefix``. + + .. versionchanged:: 0.15 + ``X-Forwarded-Host`` and ``X-Forwarded-Port`` modify + ``SERVER_NAME`` and ``SERVER_PORT``. + """ + + def __init__( + self, + app: WSGIApplication, + x_for: int = 1, + x_proto: int = 1, + x_host: int = 0, + x_port: int = 0, + x_prefix: int = 0, + ) -> None: + self.app = app + self.x_for = x_for + self.x_proto = x_proto + self.x_host = x_host + self.x_port = x_port + self.x_prefix = x_prefix + + def _get_real_value(self, trusted: int, value: str | None) -> str | None: + """Get the real value from a list header based on the configured + number of trusted proxies. + + :param trusted: Number of values to trust in the header. + :param value: Comma separated list header value to parse. + :return: The real value, or ``None`` if there are fewer values + than the number of trusted proxies. + + .. versionchanged:: 1.0 + Renamed from ``_get_trusted_comma``. + + .. versionadded:: 0.15 + """ + if not (trusted and value): + return None + values = parse_list_header(value) + if len(values) >= trusted: + return values[-trusted] + return None + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + """Modify the WSGI environ based on the various ``Forwarded`` + headers before calling the wrapped application. Store the + original environ values in ``werkzeug.proxy_fix.orig_{key}``. + """ + environ_get = environ.get + orig_remote_addr = environ_get("REMOTE_ADDR") + orig_wsgi_url_scheme = environ_get("wsgi.url_scheme") + orig_http_host = environ_get("HTTP_HOST") + environ.update( + { + "werkzeug.proxy_fix.orig": { + "REMOTE_ADDR": orig_remote_addr, + "wsgi.url_scheme": orig_wsgi_url_scheme, + "HTTP_HOST": orig_http_host, + "SERVER_NAME": environ_get("SERVER_NAME"), + "SERVER_PORT": environ_get("SERVER_PORT"), + "SCRIPT_NAME": environ_get("SCRIPT_NAME"), + } + } + ) + + x_for = self._get_real_value(self.x_for, environ_get("HTTP_X_FORWARDED_FOR")) + if x_for: + environ["REMOTE_ADDR"] = x_for + + x_proto = self._get_real_value( + self.x_proto, environ_get("HTTP_X_FORWARDED_PROTO") + ) + if x_proto: + environ["wsgi.url_scheme"] = x_proto + + x_host = self._get_real_value(self.x_host, environ_get("HTTP_X_FORWARDED_HOST")) + if x_host: + environ["HTTP_HOST"] = environ["SERVER_NAME"] = x_host + # "]" to check for IPv6 address without port + if ":" in x_host and not x_host.endswith("]"): + environ["SERVER_NAME"], environ["SERVER_PORT"] = x_host.rsplit(":", 1) + + x_port = self._get_real_value(self.x_port, environ_get("HTTP_X_FORWARDED_PORT")) + if x_port: + host = environ.get("HTTP_HOST") + if host: + # "]" to check for IPv6 address without port + if ":" in host and not host.endswith("]"): + host = host.rsplit(":", 1)[0] + environ["HTTP_HOST"] = f"{host}:{x_port}" + environ["SERVER_PORT"] = x_port + + x_prefix = self._get_real_value( + self.x_prefix, environ_get("HTTP_X_FORWARDED_PREFIX") + ) + if x_prefix: + environ["SCRIPT_NAME"] = x_prefix + + return self.app(environ, start_response) diff --git a/lib/python3.12/site-packages/werkzeug/middleware/shared_data.py b/lib/python3.12/site-packages/werkzeug/middleware/shared_data.py new file mode 100644 index 0000000000000000000000000000000000000000..c7c06df5a4f0bfeb6f94785804245df0e2c645ee --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/middleware/shared_data.py @@ -0,0 +1,283 @@ +""" +Serve Shared Static Files +========================= + +.. autoclass:: SharedDataMiddleware + :members: is_allowed + +:copyright: 2007 Pallets +:license: BSD-3-Clause +""" + +from __future__ import annotations + +import collections.abc as cabc +import importlib.util +import mimetypes +import os +import posixpath +import typing as t +from datetime import datetime +from datetime import timezone +from io import BytesIO +from time import time +from zlib import adler32 + +from ..http import http_date +from ..http import is_resource_modified +from ..security import safe_join +from ..utils import get_content_type +from ..wsgi import get_path_info +from ..wsgi import wrap_file + +_TOpener = t.Callable[[], tuple[t.IO[bytes], datetime, int]] +_TLoader = t.Callable[[t.Optional[str]], tuple[t.Optional[str], t.Optional[_TOpener]]] + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +class SharedDataMiddleware: + """A WSGI middleware which provides static content for development + environments or simple server setups. Its usage is quite simple:: + + import os + from werkzeug.middleware.shared_data import SharedDataMiddleware + + app = SharedDataMiddleware(app, { + '/shared': os.path.join(os.path.dirname(__file__), 'shared') + }) + + The contents of the folder ``./shared`` will now be available on + ``http://example.com/shared/``. This is pretty useful during development + because a standalone media server is not required. Files can also be + mounted on the root folder and still continue to use the application because + the shared data middleware forwards all unhandled requests to the + application, even if the requests are below one of the shared folders. + + If `pkg_resources` is available you can also tell the middleware to serve + files from package data:: + + app = SharedDataMiddleware(app, { + '/static': ('myapplication', 'static') + }) + + This will then serve the ``static`` folder in the `myapplication` + Python package. + + The optional `disallow` parameter can be a list of :func:`~fnmatch.fnmatch` + rules for files that are not accessible from the web. If `cache` is set to + `False` no caching headers are sent. + + Currently the middleware does not support non-ASCII filenames. If the + encoding on the file system happens to match the encoding of the URI it may + work but this could also be by accident. We strongly suggest using ASCII + only file names for static files. + + The middleware will guess the mimetype using the Python `mimetype` + module. If it's unable to figure out the charset it will fall back + to `fallback_mimetype`. + + :param app: the application to wrap. If you don't want to wrap an + application you can pass it :exc:`NotFound`. + :param exports: a list or dict of exported files and folders. + :param disallow: a list of :func:`~fnmatch.fnmatch` rules. + :param cache: enable or disable caching headers. + :param cache_timeout: the cache timeout in seconds for the headers. + :param fallback_mimetype: The fallback mimetype for unknown files. + + .. versionchanged:: 1.0 + The default ``fallback_mimetype`` is + ``application/octet-stream``. If a filename looks like a text + mimetype, the ``utf-8`` charset is added to it. + + .. versionadded:: 0.6 + Added ``fallback_mimetype``. + + .. versionchanged:: 0.5 + Added ``cache_timeout``. + """ + + def __init__( + self, + app: WSGIApplication, + exports: ( + cabc.Mapping[str, str | tuple[str, str]] + | t.Iterable[tuple[str, str | tuple[str, str]]] + ), + disallow: None = None, + cache: bool = True, + cache_timeout: int = 60 * 60 * 12, + fallback_mimetype: str = "application/octet-stream", + ) -> None: + self.app = app + self.exports: list[tuple[str, _TLoader]] = [] + self.cache = cache + self.cache_timeout = cache_timeout + + if isinstance(exports, cabc.Mapping): + exports = exports.items() + + for key, value in exports: + if isinstance(value, tuple): + loader = self.get_package_loader(*value) + elif isinstance(value, str): + if os.path.isfile(value): + loader = self.get_file_loader(value) + else: + loader = self.get_directory_loader(value) + else: + raise TypeError(f"unknown def {value!r}") + + self.exports.append((key, loader)) + + if disallow is not None: + from fnmatch import fnmatch + + self.is_allowed = lambda x: not fnmatch(x, disallow) + + self.fallback_mimetype = fallback_mimetype + + def is_allowed(self, filename: str) -> bool: + """Subclasses can override this method to disallow the access to + certain files. However by providing `disallow` in the constructor + this method is overwritten. + """ + return True + + def _opener(self, filename: str) -> _TOpener: + return lambda: ( + open(filename, "rb"), + datetime.fromtimestamp(os.path.getmtime(filename), tz=timezone.utc), + int(os.path.getsize(filename)), + ) + + def get_file_loader(self, filename: str) -> _TLoader: + return lambda x: (os.path.basename(filename), self._opener(filename)) + + def get_package_loader(self, package: str, package_path: str) -> _TLoader: + load_time = datetime.now(timezone.utc) + spec = importlib.util.find_spec(package) + reader = spec.loader.get_resource_reader(package) # type: ignore[union-attr] + + def loader( + path: str | None, + ) -> tuple[str | None, _TOpener | None]: + if path is None: + return None, None + + path = safe_join(package_path, path) + + if path is None: + return None, None + + basename = posixpath.basename(path) + + try: + resource = reader.open_resource(path) + except OSError: + return None, None + + if isinstance(resource, BytesIO): + return ( + basename, + lambda: (resource, load_time, len(resource.getvalue())), + ) + + return ( + basename, + lambda: ( + resource, + datetime.fromtimestamp( + os.path.getmtime(resource.name), tz=timezone.utc + ), + os.path.getsize(resource.name), + ), + ) + + return loader + + def get_directory_loader(self, directory: str) -> _TLoader: + def loader( + path: str | None, + ) -> tuple[str | None, _TOpener | None]: + if path is not None: + path = safe_join(directory, path) + + if path is None: + return None, None + else: + path = directory + + if os.path.isfile(path): + return os.path.basename(path), self._opener(path) + + return None, None + + return loader + + def generate_etag(self, mtime: datetime, file_size: int, real_filename: str) -> str: + fn_str = os.fsencode(real_filename) + timestamp = mtime.timestamp() + checksum = adler32(fn_str) & 0xFFFFFFFF + return f"wzsdm-{timestamp}-{file_size}-{checksum}" + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + path = get_path_info(environ) + file_loader = None + + for search_path, loader in self.exports: + if search_path == path: + real_filename, file_loader = loader(None) + + if file_loader is not None: + break + + if not search_path.endswith("/"): + search_path += "/" + + if path.startswith(search_path): + real_filename, file_loader = loader(path[len(search_path) :]) + + if file_loader is not None: + break + + if file_loader is None or not self.is_allowed(real_filename): # type: ignore + return self.app(environ, start_response) + + guessed_type = mimetypes.guess_type(real_filename) # type: ignore + mime_type = get_content_type(guessed_type[0] or self.fallback_mimetype, "utf-8") + f, mtime, file_size = file_loader() + + headers = [("Date", http_date())] + + if self.cache: + timeout = self.cache_timeout + etag = self.generate_etag(mtime, file_size, real_filename) # type: ignore + headers += [ + ("Etag", f'"{etag}"'), + ("Cache-Control", f"max-age={timeout}, public"), + ] + + if not is_resource_modified(environ, etag, last_modified=mtime): + f.close() + start_response("304 Not Modified", headers) + return [] + + headers.append(("Expires", http_date(time() + timeout))) + else: + headers.append(("Cache-Control", "public")) + + headers.extend( + ( + ("Content-Type", mime_type), + ("Content-Length", str(file_size)), + ("Last-Modified", http_date(mtime)), + ) + ) + start_response("200 OK", headers) + return wrap_file(environ, f) diff --git a/lib/python3.12/site-packages/werkzeug/py.typed b/lib/python3.12/site-packages/werkzeug/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/werkzeug/routing/__init__.py b/lib/python3.12/site-packages/werkzeug/routing/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..62adc48fb03b5500e1e28f6887cc6ed582beb387 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/routing/__init__.py @@ -0,0 +1,134 @@ +"""When it comes to combining multiple controller or view functions +(however you want to call them) you need a dispatcher. A simple way +would be applying regular expression tests on the ``PATH_INFO`` and +calling registered callback functions that return the value then. + +This module implements a much more powerful system than simple regular +expression matching because it can also convert values in the URLs and +build URLs. + +Here a simple example that creates a URL map for an application with +two subdomains (www and kb) and some URL rules: + +.. code-block:: python + + m = Map([ + # Static URLs + Rule('/', endpoint='static/index'), + Rule('/about', endpoint='static/about'), + Rule('/help', endpoint='static/help'), + # Knowledge Base + Subdomain('kb', [ + Rule('/', endpoint='kb/index'), + Rule('/browse/', endpoint='kb/browse'), + Rule('/browse//', endpoint='kb/browse'), + Rule('/browse//', endpoint='kb/browse') + ]) + ], default_subdomain='www') + +If the application doesn't use subdomains it's perfectly fine to not set +the default subdomain and not use the `Subdomain` rule factory. The +endpoint in the rules can be anything, for example import paths or +unique identifiers. The WSGI application can use those endpoints to get the +handler for that URL. It doesn't have to be a string at all but it's +recommended. + +Now it's possible to create a URL adapter for one of the subdomains and +build URLs: + +.. code-block:: python + + c = m.bind('example.com') + + c.build("kb/browse", dict(id=42)) + 'http://kb.example.com/browse/42/' + + c.build("kb/browse", dict()) + 'http://kb.example.com/browse/' + + c.build("kb/browse", dict(id=42, page=3)) + 'http://kb.example.com/browse/42/3' + + c.build("static/about") + '/about' + + c.build("static/index", force_external=True) + 'http://www.example.com/' + + c = m.bind('example.com', subdomain='kb') + + c.build("static/about") + 'http://www.example.com/about' + +The first argument to bind is the server name *without* the subdomain. +Per default it will assume that the script is mounted on the root, but +often that's not the case so you can provide the real mount point as +second argument: + +.. code-block:: python + + c = m.bind('example.com', '/applications/example') + +The third argument can be the subdomain, if not given the default +subdomain is used. For more details about binding have a look at the +documentation of the `MapAdapter`. + +And here is how you can match URLs: + +.. code-block:: python + + c = m.bind('example.com') + + c.match("/") + ('static/index', {}) + + c.match("/about") + ('static/about', {}) + + c = m.bind('example.com', '/', 'kb') + + c.match("/") + ('kb/index', {}) + + c.match("/browse/42/23") + ('kb/browse', {'id': 42, 'page': 23}) + +If matching fails you get a ``NotFound`` exception, if the rule thinks +it's a good idea to redirect (for example because the URL was defined +to have a slash at the end but the request was missing that slash) it +will raise a ``RequestRedirect`` exception. Both are subclasses of +``HTTPException`` so you can use those errors as responses in the +application. + +If matching succeeded but the URL rule was incompatible to the given +method (for example there were only rules for ``GET`` and ``HEAD`` but +routing tried to match a ``POST`` request) a ``MethodNotAllowed`` +exception is raised. +""" + +from .converters import AnyConverter as AnyConverter +from .converters import BaseConverter as BaseConverter +from .converters import FloatConverter as FloatConverter +from .converters import IntegerConverter as IntegerConverter +from .converters import PathConverter as PathConverter +from .converters import UnicodeConverter as UnicodeConverter +from .converters import UUIDConverter as UUIDConverter +from .converters import ValidationError as ValidationError +from .exceptions import BuildError as BuildError +from .exceptions import NoMatch as NoMatch +from .exceptions import RequestAliasRedirect as RequestAliasRedirect +from .exceptions import RequestPath as RequestPath +from .exceptions import RequestRedirect as RequestRedirect +from .exceptions import RoutingException as RoutingException +from .exceptions import WebsocketMismatch as WebsocketMismatch +from .map import Map as Map +from .map import MapAdapter as MapAdapter +from .matcher import StateMachineMatcher as StateMachineMatcher +from .rules import EndpointPrefix as EndpointPrefix +from .rules import parse_converter_args as parse_converter_args +from .rules import Rule as Rule +from .rules import RuleFactory as RuleFactory +from .rules import RuleTemplate as RuleTemplate +from .rules import RuleTemplateFactory as RuleTemplateFactory +from .rules import Subdomain as Subdomain +from .rules import Submount as Submount diff --git a/lib/python3.12/site-packages/werkzeug/routing/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..32e399cd9c45d2270b0ca24f0cc3743793a23c5d Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/routing/__pycache__/converters.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/converters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3135c082d19921ff5a2473481f2203499d00b06f Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/converters.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/routing/__pycache__/exceptions.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48104660b260207a489add1de1d82c1e48dee86e Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/exceptions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/routing/__pycache__/map.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/map.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f69729825dfb0c5f6150aa2be13b38a46ea4ad99 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/map.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/routing/__pycache__/matcher.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/matcher.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4d31285ef75b4bfa49b95abbbe95716ba776b463 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/matcher.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/routing/__pycache__/rules.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/rules.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb55fe8de2723c1979297e17c4fa9d9c6fe9caa0 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/routing/__pycache__/rules.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/routing/converters.py b/lib/python3.12/site-packages/werkzeug/routing/converters.py new file mode 100644 index 0000000000000000000000000000000000000000..6016a975edf0f2e830728d906f268cab336b8c0e --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/routing/converters.py @@ -0,0 +1,261 @@ +from __future__ import annotations + +import re +import typing as t +import uuid +from urllib.parse import quote + +if t.TYPE_CHECKING: + from .map import Map + + +class ValidationError(ValueError): + """Validation error. If a rule converter raises this exception the rule + does not match the current URL and the next URL is tried. + """ + + +class BaseConverter: + """Base class for all converters. + + .. versionchanged:: 2.3 + ``part_isolating`` defaults to ``False`` if ``regex`` contains a ``/``. + """ + + regex = "[^/]+" + weight = 100 + part_isolating = True + + def __init_subclass__(cls, **kwargs: t.Any) -> None: + super().__init_subclass__(**kwargs) + + # If the converter isn't inheriting its regex, disable part_isolating by default + # if the regex contains a / character. + if "regex" in cls.__dict__ and "part_isolating" not in cls.__dict__: + cls.part_isolating = "/" not in cls.regex + + def __init__(self, map: Map, *args: t.Any, **kwargs: t.Any) -> None: + self.map = map + + def to_python(self, value: str) -> t.Any: + return value + + def to_url(self, value: t.Any) -> str: + # safe = https://url.spec.whatwg.org/#url-path-segment-string + return quote(str(value), safe="!$&'()*+,/:;=@") + + +class UnicodeConverter(BaseConverter): + """This converter is the default converter and accepts any string but + only one path segment. Thus the string can not include a slash. + + This is the default validator. + + Example:: + + Rule('/pages/'), + Rule('/') + + :param map: the :class:`Map`. + :param minlength: the minimum length of the string. Must be greater + or equal 1. + :param maxlength: the maximum length of the string. + :param length: the exact length of the string. + """ + + def __init__( + self, + map: Map, + minlength: int = 1, + maxlength: int | None = None, + length: int | None = None, + ) -> None: + super().__init__(map) + if length is not None: + length_regex = f"{{{int(length)}}}" + else: + if maxlength is None: + maxlength_value = "" + else: + maxlength_value = str(int(maxlength)) + length_regex = f"{{{int(minlength)},{maxlength_value}}}" + self.regex = f"[^/]{length_regex}" + + +class AnyConverter(BaseConverter): + """Matches one of the items provided. Items can either be Python + identifiers or strings:: + + Rule('/') + + :param map: the :class:`Map`. + :param items: this function accepts the possible items as positional + arguments. + + .. versionchanged:: 2.2 + Value is validated when building a URL. + """ + + def __init__(self, map: Map, *items: str) -> None: + super().__init__(map) + self.items = set(items) + self.regex = f"(?:{'|'.join([re.escape(x) for x in items])})" + + def to_url(self, value: t.Any) -> str: + if value in self.items: + return str(value) + + valid_values = ", ".join(f"'{item}'" for item in sorted(self.items)) + raise ValueError(f"'{value}' is not one of {valid_values}") + + +class PathConverter(BaseConverter): + """Like the default :class:`UnicodeConverter`, but it also matches + slashes. This is useful for wikis and similar applications:: + + Rule('/') + Rule('//edit') + + :param map: the :class:`Map`. + """ + + part_isolating = False + regex = "[^/].*?" + weight = 200 + + +class NumberConverter(BaseConverter): + """Baseclass for `IntegerConverter` and `FloatConverter`. + + :internal: + """ + + weight = 50 + num_convert: t.Callable[[t.Any], t.Any] = int + + def __init__( + self, + map: Map, + fixed_digits: int = 0, + min: int | None = None, + max: int | None = None, + signed: bool = False, + ) -> None: + if signed: + self.regex = self.signed_regex + super().__init__(map) + self.fixed_digits = fixed_digits + self.min = min + self.max = max + self.signed = signed + + def to_python(self, value: str) -> t.Any: + if self.fixed_digits and len(value) != self.fixed_digits: + raise ValidationError() + value_num = self.num_convert(value) + if (self.min is not None and value_num < self.min) or ( + self.max is not None and value_num > self.max + ): + raise ValidationError() + return value_num + + def to_url(self, value: t.Any) -> str: + value_str = str(self.num_convert(value)) + if self.fixed_digits: + value_str = value_str.zfill(self.fixed_digits) + return value_str + + @property + def signed_regex(self) -> str: + return f"-?{self.regex}" + + +class IntegerConverter(NumberConverter): + """This converter only accepts integer values:: + + Rule("/page/") + + By default it only accepts unsigned, positive values. The ``signed`` + parameter will enable signed, negative values. :: + + Rule("/page/") + + :param map: The :class:`Map`. + :param fixed_digits: The number of fixed digits in the URL. If you + set this to ``4`` for example, the rule will only match if the + URL looks like ``/0001/``. The default is variable length. + :param min: The minimal value. + :param max: The maximal value. + :param signed: Allow signed (negative) values. + + .. versionadded:: 0.15 + The ``signed`` parameter. + """ + + regex = r"\d+" + + +class FloatConverter(NumberConverter): + """This converter only accepts floating point values:: + + Rule("/probability/") + + By default it only accepts unsigned, positive values. The ``signed`` + parameter will enable signed, negative values. :: + + Rule("/offset/") + + :param map: The :class:`Map`. + :param min: The minimal value. + :param max: The maximal value. + :param signed: Allow signed (negative) values. + + .. versionadded:: 0.15 + The ``signed`` parameter. + """ + + regex = r"\d+\.\d+" + num_convert = float + + def __init__( + self, + map: Map, + min: float | None = None, + max: float | None = None, + signed: bool = False, + ) -> None: + super().__init__(map, min=min, max=max, signed=signed) # type: ignore + + +class UUIDConverter(BaseConverter): + """This converter only accepts UUID strings:: + + Rule('/object/') + + .. versionadded:: 0.10 + + :param map: the :class:`Map`. + """ + + regex = ( + r"[A-Fa-f0-9]{8}-[A-Fa-f0-9]{4}-" + r"[A-Fa-f0-9]{4}-[A-Fa-f0-9]{4}-[A-Fa-f0-9]{12}" + ) + + def to_python(self, value: str) -> uuid.UUID: + return uuid.UUID(value) + + def to_url(self, value: uuid.UUID) -> str: + return str(value) + + +#: the default converter mapping for the map. +DEFAULT_CONVERTERS: t.Mapping[str, type[BaseConverter]] = { + "default": UnicodeConverter, + "string": UnicodeConverter, + "any": AnyConverter, + "path": PathConverter, + "int": IntegerConverter, + "float": FloatConverter, + "uuid": UUIDConverter, +} diff --git a/lib/python3.12/site-packages/werkzeug/routing/exceptions.py b/lib/python3.12/site-packages/werkzeug/routing/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..eeabd4ed12f6828778c55588e6ddccd0b1f9240c --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/routing/exceptions.py @@ -0,0 +1,152 @@ +from __future__ import annotations + +import difflib +import typing as t + +from ..exceptions import BadRequest +from ..exceptions import HTTPException +from ..utils import cached_property +from ..utils import redirect + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIEnvironment + + from ..wrappers.request import Request + from ..wrappers.response import Response + from .map import MapAdapter + from .rules import Rule + + +class RoutingException(Exception): + """Special exceptions that require the application to redirect, notifying + about missing urls, etc. + + :internal: + """ + + +class RequestRedirect(HTTPException, RoutingException): + """Raise if the map requests a redirect. This is for example the case if + `strict_slashes` are activated and an url that requires a trailing slash. + + The attribute `new_url` contains the absolute destination url. + """ + + code = 308 + + def __init__(self, new_url: str) -> None: + super().__init__(new_url) + self.new_url = new_url + + def get_response( + self, + environ: WSGIEnvironment | Request | None = None, + scope: dict[str, t.Any] | None = None, + ) -> Response: + return redirect(self.new_url, self.code) + + +class RequestPath(RoutingException): + """Internal exception.""" + + __slots__ = ("path_info",) + + def __init__(self, path_info: str) -> None: + super().__init__() + self.path_info = path_info + + +class RequestAliasRedirect(RoutingException): # noqa: B903 + """This rule is an alias and wants to redirect to the canonical URL.""" + + def __init__(self, matched_values: t.Mapping[str, t.Any], endpoint: t.Any) -> None: + super().__init__() + self.matched_values = matched_values + self.endpoint = endpoint + + +class BuildError(RoutingException, LookupError): + """Raised if the build system cannot find a URL for an endpoint with the + values provided. + """ + + def __init__( + self, + endpoint: t.Any, + values: t.Mapping[str, t.Any], + method: str | None, + adapter: MapAdapter | None = None, + ) -> None: + super().__init__(endpoint, values, method) + self.endpoint = endpoint + self.values = values + self.method = method + self.adapter = adapter + + @cached_property + def suggested(self) -> Rule | None: + return self.closest_rule(self.adapter) + + def closest_rule(self, adapter: MapAdapter | None) -> Rule | None: + def _score_rule(rule: Rule) -> float: + return sum( + [ + 0.98 + * difflib.SequenceMatcher( + # endpoints can be any type, compare as strings + None, + str(rule.endpoint), + str(self.endpoint), + ).ratio(), + 0.01 * bool(set(self.values or ()).issubset(rule.arguments)), + 0.01 * bool(rule.methods and self.method in rule.methods), + ] + ) + + if adapter and adapter.map._rules: + return max(adapter.map._rules, key=_score_rule) + + return None + + def __str__(self) -> str: + message = [f"Could not build url for endpoint {self.endpoint!r}"] + if self.method: + message.append(f" ({self.method!r})") + if self.values: + message.append(f" with values {sorted(self.values)!r}") + message.append(".") + if self.suggested: + if self.endpoint == self.suggested.endpoint: + if ( + self.method + and self.suggested.methods is not None + and self.method not in self.suggested.methods + ): + message.append( + " Did you mean to use methods" + f" {sorted(self.suggested.methods)!r}?" + ) + missing_values = self.suggested.arguments.union( + set(self.suggested.defaults or ()) + ) - set(self.values.keys()) + if missing_values: + message.append( + f" Did you forget to specify values {sorted(missing_values)!r}?" + ) + else: + message.append(f" Did you mean {self.suggested.endpoint!r} instead?") + return "".join(message) + + +class WebsocketMismatch(BadRequest): + """The only matched rule is either a WebSocket and the request is + HTTP, or the rule is HTTP and the request is a WebSocket. + """ + + +class NoMatch(Exception): + __slots__ = ("have_match_for", "websocket_mismatch") + + def __init__(self, have_match_for: set[str], websocket_mismatch: bool) -> None: + self.have_match_for = have_match_for + self.websocket_mismatch = websocket_mismatch diff --git a/lib/python3.12/site-packages/werkzeug/routing/map.py b/lib/python3.12/site-packages/werkzeug/routing/map.py new file mode 100644 index 0000000000000000000000000000000000000000..f757d59b948c999237647d5f039599fb2d569fa7 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/routing/map.py @@ -0,0 +1,951 @@ +from __future__ import annotations + +import typing as t +import warnings +from pprint import pformat +from threading import Lock +from urllib.parse import quote +from urllib.parse import urljoin +from urllib.parse import urlunsplit + +from .._internal import _get_environ +from .._internal import _wsgi_decoding_dance +from ..datastructures import ImmutableDict +from ..datastructures import MultiDict +from ..exceptions import BadHost +from ..exceptions import HTTPException +from ..exceptions import MethodNotAllowed +from ..exceptions import NotFound +from ..urls import _urlencode +from ..wsgi import get_host +from .converters import DEFAULT_CONVERTERS +from .exceptions import BuildError +from .exceptions import NoMatch +from .exceptions import RequestAliasRedirect +from .exceptions import RequestPath +from .exceptions import RequestRedirect +from .exceptions import WebsocketMismatch +from .matcher import StateMachineMatcher +from .rules import _simple_rule_re +from .rules import Rule + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + from ..wrappers.request import Request + from .converters import BaseConverter + from .rules import RuleFactory + + +class Map: + """The map class stores all the URL rules and some configuration + parameters. Some of the configuration values are only stored on the + `Map` instance since those affect all rules, others are just defaults + and can be overridden for each rule. Note that you have to specify all + arguments besides the `rules` as keyword arguments! + + :param rules: sequence of url rules for this map. + :param default_subdomain: The default subdomain for rules without a + subdomain defined. + :param strict_slashes: If a rule ends with a slash but the matched + URL does not, redirect to the URL with a trailing slash. + :param merge_slashes: Merge consecutive slashes when matching or + building URLs. Matches will redirect to the normalized URL. + Slashes in variable parts are not merged. + :param redirect_defaults: This will redirect to the default rule if it + wasn't visited that way. This helps creating + unique URLs. + :param converters: A dict of converters that adds additional converters + to the list of converters. If you redefine one + converter this will override the original one. + :param sort_parameters: If set to `True` the url parameters are sorted. + See `url_encode` for more details. + :param sort_key: The sort key function for `url_encode`. + :param host_matching: if set to `True` it enables the host matching + feature and disables the subdomain one. If + enabled the `host` parameter to rules is used + instead of the `subdomain` one. + + .. versionchanged:: 3.0 + The ``charset`` and ``encoding_errors`` parameters were removed. + + .. versionchanged:: 1.0 + If ``url_scheme`` is ``ws`` or ``wss``, only WebSocket rules will match. + + .. versionchanged:: 1.0 + The ``merge_slashes`` parameter was added. + + .. versionchanged:: 0.7 + The ``encoding_errors`` and ``host_matching`` parameters were added. + + .. versionchanged:: 0.5 + The ``sort_parameters`` and ``sort_key`` parameters were added. + """ + + #: A dict of default converters to be used. + default_converters = ImmutableDict(DEFAULT_CONVERTERS) + + #: The type of lock to use when updating. + #: + #: .. versionadded:: 1.0 + lock_class = Lock + + def __init__( + self, + rules: t.Iterable[RuleFactory] | None = None, + default_subdomain: str = "", + strict_slashes: bool = True, + merge_slashes: bool = True, + redirect_defaults: bool = True, + converters: t.Mapping[str, type[BaseConverter]] | None = None, + sort_parameters: bool = False, + sort_key: t.Callable[[t.Any], t.Any] | None = None, + host_matching: bool = False, + ) -> None: + self._matcher = StateMachineMatcher(merge_slashes) + self._rules_by_endpoint: dict[t.Any, list[Rule]] = {} + self._remap = True + self._remap_lock = self.lock_class() + + self.default_subdomain = default_subdomain + self.strict_slashes = strict_slashes + self.redirect_defaults = redirect_defaults + self.host_matching = host_matching + + self.converters = self.default_converters.copy() + if converters: + self.converters.update(converters) + + self.sort_parameters = sort_parameters + self.sort_key = sort_key + + for rulefactory in rules or (): + self.add(rulefactory) + + @property + def merge_slashes(self) -> bool: + return self._matcher.merge_slashes + + @merge_slashes.setter + def merge_slashes(self, value: bool) -> None: + self._matcher.merge_slashes = value + + def is_endpoint_expecting(self, endpoint: t.Any, *arguments: str) -> bool: + """Iterate over all rules and check if the endpoint expects + the arguments provided. This is for example useful if you have + some URLs that expect a language code and others that do not and + you want to wrap the builder a bit so that the current language + code is automatically added if not provided but endpoints expect + it. + + :param endpoint: the endpoint to check. + :param arguments: this function accepts one or more arguments + as positional arguments. Each one of them is + checked. + """ + self.update() + arguments_set = set(arguments) + for rule in self._rules_by_endpoint[endpoint]: + if arguments_set.issubset(rule.arguments): + return True + return False + + @property + def _rules(self) -> list[Rule]: + return [rule for rules in self._rules_by_endpoint.values() for rule in rules] + + def iter_rules(self, endpoint: t.Any | None = None) -> t.Iterator[Rule]: + """Iterate over all rules or the rules of an endpoint. + + :param endpoint: if provided only the rules for that endpoint + are returned. + :return: an iterator + """ + self.update() + if endpoint is not None: + return iter(self._rules_by_endpoint[endpoint]) + return iter(self._rules) + + def add(self, rulefactory: RuleFactory) -> None: + """Add a new rule or factory to the map and bind it. Requires that the + rule is not bound to another map. + + :param rulefactory: a :class:`Rule` or :class:`RuleFactory` + """ + for rule in rulefactory.get_rules(self): + rule.bind(self) + if not rule.build_only: + self._matcher.add(rule) + self._rules_by_endpoint.setdefault(rule.endpoint, []).append(rule) + self._remap = True + + def bind( + self, + server_name: str, + script_name: str | None = None, + subdomain: str | None = None, + url_scheme: str = "http", + default_method: str = "GET", + path_info: str | None = None, + query_args: t.Mapping[str, t.Any] | str | None = None, + ) -> MapAdapter: + """Return a new :class:`MapAdapter` with the details specified to the + call. Note that `script_name` will default to ``'/'`` if not further + specified or `None`. The `server_name` at least is a requirement + because the HTTP RFC requires absolute URLs for redirects and so all + redirect exceptions raised by Werkzeug will contain the full canonical + URL. + + If no path_info is passed to :meth:`match` it will use the default path + info passed to bind. While this doesn't really make sense for + manual bind calls, it's useful if you bind a map to a WSGI + environment which already contains the path info. + + `subdomain` will default to the `default_subdomain` for this map if + no defined. If there is no `default_subdomain` you cannot use the + subdomain feature. + + .. versionchanged:: 1.0 + If ``url_scheme`` is ``ws`` or ``wss``, only WebSocket rules + will match. + + .. versionchanged:: 0.15 + ``path_info`` defaults to ``'/'`` if ``None``. + + .. versionchanged:: 0.8 + ``query_args`` can be a string. + + .. versionchanged:: 0.7 + Added ``query_args``. + """ + server_name = server_name.lower() + if self.host_matching: + if subdomain is not None: + raise RuntimeError("host matching enabled and a subdomain was provided") + elif subdomain is None: + subdomain = self.default_subdomain + if script_name is None: + script_name = "/" + if path_info is None: + path_info = "/" + + # Port isn't part of IDNA, and might push a name over the 63 octet limit. + server_name, port_sep, port = server_name.partition(":") + + try: + server_name = server_name.encode("idna").decode("ascii") + except UnicodeError as e: + raise BadHost() from e + + return MapAdapter( + self, + f"{server_name}{port_sep}{port}", + script_name, + subdomain, + url_scheme, + path_info, + default_method, + query_args, + ) + + def bind_to_environ( + self, + environ: WSGIEnvironment | Request, + server_name: str | None = None, + subdomain: str | None = None, + ) -> MapAdapter: + """Like :meth:`bind` but you can pass it an WSGI environment and it + will fetch the information from that dictionary. Note that because of + limitations in the protocol there is no way to get the current + subdomain and real `server_name` from the environment. If you don't + provide it, Werkzeug will use `SERVER_NAME` and `SERVER_PORT` (or + `HTTP_HOST` if provided) as used `server_name` with disabled subdomain + feature. + + If `subdomain` is `None` but an environment and a server name is + provided it will calculate the current subdomain automatically. + Example: `server_name` is ``'example.com'`` and the `SERVER_NAME` + in the wsgi `environ` is ``'staging.dev.example.com'`` the calculated + subdomain will be ``'staging.dev'``. + + If the object passed as environ has an environ attribute, the value of + this attribute is used instead. This allows you to pass request + objects. Additionally `PATH_INFO` added as a default of the + :class:`MapAdapter` so that you don't have to pass the path info to + the match method. + + .. versionchanged:: 1.0.0 + If the passed server name specifies port 443, it will match + if the incoming scheme is ``https`` without a port. + + .. versionchanged:: 1.0.0 + A warning is shown when the passed server name does not + match the incoming WSGI server name. + + .. versionchanged:: 0.8 + This will no longer raise a ValueError when an unexpected server + name was passed. + + .. versionchanged:: 0.5 + previously this method accepted a bogus `calculate_subdomain` + parameter that did not have any effect. It was removed because + of that. + + :param environ: a WSGI environment. + :param server_name: an optional server name hint (see above). + :param subdomain: optionally the current subdomain (see above). + """ + env = _get_environ(environ) + wsgi_server_name = get_host(env).lower() + scheme = env["wsgi.url_scheme"] + upgrade = any( + v.strip() == "upgrade" + for v in env.get("HTTP_CONNECTION", "").lower().split(",") + ) + + if upgrade and env.get("HTTP_UPGRADE", "").lower() == "websocket": + scheme = "wss" if scheme == "https" else "ws" + + if server_name is None: + server_name = wsgi_server_name + else: + server_name = server_name.lower() + + # strip standard port to match get_host() + if scheme in {"http", "ws"} and server_name.endswith(":80"): + server_name = server_name[:-3] + elif scheme in {"https", "wss"} and server_name.endswith(":443"): + server_name = server_name[:-4] + + if subdomain is None and not self.host_matching: + cur_server_name = wsgi_server_name.split(".") + real_server_name = server_name.split(".") + offset = -len(real_server_name) + + if cur_server_name[offset:] != real_server_name: + # This can happen even with valid configs if the server was + # accessed directly by IP address under some situations. + # Instead of raising an exception like in Werkzeug 0.7 or + # earlier we go by an invalid subdomain which will result + # in a 404 error on matching. + warnings.warn( + f"Current server name {wsgi_server_name!r} doesn't match configured" + f" server name {server_name!r}", + stacklevel=2, + ) + subdomain = "" + else: + subdomain = ".".join(filter(None, cur_server_name[:offset])) + + def _get_wsgi_string(name: str) -> str | None: + val = env.get(name) + if val is not None: + return _wsgi_decoding_dance(val) + return None + + script_name = _get_wsgi_string("SCRIPT_NAME") + path_info = _get_wsgi_string("PATH_INFO") + query_args = _get_wsgi_string("QUERY_STRING") + return Map.bind( + self, + server_name, + script_name, + subdomain, + scheme, + env["REQUEST_METHOD"], + path_info, + query_args=query_args, + ) + + def update(self) -> None: + """Called before matching and building to keep the compiled rules + in the correct order after things changed. + """ + if not self._remap: + return + + with self._remap_lock: + if not self._remap: + return + + self._matcher.update() + for rules in self._rules_by_endpoint.values(): + rules.sort(key=lambda x: x.build_compare_key()) + self._remap = False + + def __repr__(self) -> str: + rules = self.iter_rules() + return f"{type(self).__name__}({pformat(list(rules))})" + + +class MapAdapter: + """Returned by :meth:`Map.bind` or :meth:`Map.bind_to_environ` and does + the URL matching and building based on runtime information. + """ + + def __init__( + self, + map: Map, + server_name: str, + script_name: str, + subdomain: str | None, + url_scheme: str, + path_info: str, + default_method: str, + query_args: t.Mapping[str, t.Any] | str | None = None, + ): + self.map = map + self.server_name = server_name + + if not script_name.endswith("/"): + script_name += "/" + + self.script_name = script_name + self.subdomain = subdomain + self.url_scheme = url_scheme + self.path_info = path_info + self.default_method = default_method + self.query_args = query_args + self.websocket = self.url_scheme in {"ws", "wss"} + + def dispatch( + self, + view_func: t.Callable[[str, t.Mapping[str, t.Any]], WSGIApplication], + path_info: str | None = None, + method: str | None = None, + catch_http_exceptions: bool = False, + ) -> WSGIApplication: + """Does the complete dispatching process. `view_func` is called with + the endpoint and a dict with the values for the view. It should + look up the view function, call it, and return a response object + or WSGI application. http exceptions are not caught by default + so that applications can display nicer error messages by just + catching them by hand. If you want to stick with the default + error messages you can pass it ``catch_http_exceptions=True`` and + it will catch the http exceptions. + + Here a small example for the dispatch usage:: + + from werkzeug.wrappers import Request, Response + from werkzeug.wsgi import responder + from werkzeug.routing import Map, Rule + + def on_index(request): + return Response('Hello from the index') + + url_map = Map([Rule('/', endpoint='index')]) + views = {'index': on_index} + + @responder + def application(environ, start_response): + request = Request(environ) + urls = url_map.bind_to_environ(environ) + return urls.dispatch(lambda e, v: views[e](request, **v), + catch_http_exceptions=True) + + Keep in mind that this method might return exception objects, too, so + use :class:`Response.force_type` to get a response object. + + :param view_func: a function that is called with the endpoint as + first argument and the value dict as second. Has + to dispatch to the actual view function with this + information. (see above) + :param path_info: the path info to use for matching. Overrides the + path info specified on binding. + :param method: the HTTP method used for matching. Overrides the + method specified on binding. + :param catch_http_exceptions: set to `True` to catch any of the + werkzeug :class:`HTTPException`\\s. + """ + try: + try: + endpoint, args = self.match(path_info, method) + except RequestRedirect as e: + return e + return view_func(endpoint, args) + except HTTPException as e: + if catch_http_exceptions: + return e + raise + + @t.overload + def match( + self, + path_info: str | None = None, + method: str | None = None, + return_rule: t.Literal[False] = False, + query_args: t.Mapping[str, t.Any] | str | None = None, + websocket: bool | None = None, + ) -> tuple[t.Any, t.Mapping[str, t.Any]]: ... + + @t.overload + def match( + self, + path_info: str | None = None, + method: str | None = None, + return_rule: t.Literal[True] = True, + query_args: t.Mapping[str, t.Any] | str | None = None, + websocket: bool | None = None, + ) -> tuple[Rule, t.Mapping[str, t.Any]]: ... + + def match( + self, + path_info: str | None = None, + method: str | None = None, + return_rule: bool = False, + query_args: t.Mapping[str, t.Any] | str | None = None, + websocket: bool | None = None, + ) -> tuple[t.Any | Rule, t.Mapping[str, t.Any]]: + """The usage is simple: you just pass the match method the current + path info as well as the method (which defaults to `GET`). The + following things can then happen: + + - you receive a `NotFound` exception that indicates that no URL is + matching. A `NotFound` exception is also a WSGI application you + can call to get a default page not found page (happens to be the + same object as `werkzeug.exceptions.NotFound`) + + - you receive a `MethodNotAllowed` exception that indicates that there + is a match for this URL but not for the current request method. + This is useful for RESTful applications. + + - you receive a `RequestRedirect` exception with a `new_url` + attribute. This exception is used to notify you about a request + Werkzeug requests from your WSGI application. This is for example the + case if you request ``/foo`` although the correct URL is ``/foo/`` + You can use the `RequestRedirect` instance as response-like object + similar to all other subclasses of `HTTPException`. + + - you receive a ``WebsocketMismatch`` exception if the only + match is a WebSocket rule but the bind is an HTTP request, or + if the match is an HTTP rule but the bind is a WebSocket + request. + + - you get a tuple in the form ``(endpoint, arguments)`` if there is + a match (unless `return_rule` is True, in which case you get a tuple + in the form ``(rule, arguments)``) + + If the path info is not passed to the match method the default path + info of the map is used (defaults to the root URL if not defined + explicitly). + + All of the exceptions raised are subclasses of `HTTPException` so they + can be used as WSGI responses. They will all render generic error or + redirect pages. + + Here is a small example for matching: + + >>> m = Map([ + ... Rule('/', endpoint='index'), + ... Rule('/downloads/', endpoint='downloads/index'), + ... Rule('/downloads/', endpoint='downloads/show') + ... ]) + >>> urls = m.bind("example.com", "/") + >>> urls.match("/", "GET") + ('index', {}) + >>> urls.match("/downloads/42") + ('downloads/show', {'id': 42}) + + And here is what happens on redirect and missing URLs: + + >>> urls.match("/downloads") + Traceback (most recent call last): + ... + RequestRedirect: http://example.com/downloads/ + >>> urls.match("/missing") + Traceback (most recent call last): + ... + NotFound: 404 Not Found + + :param path_info: the path info to use for matching. Overrides the + path info specified on binding. + :param method: the HTTP method used for matching. Overrides the + method specified on binding. + :param return_rule: return the rule that matched instead of just the + endpoint (defaults to `False`). + :param query_args: optional query arguments that are used for + automatic redirects as string or dictionary. It's + currently not possible to use the query arguments + for URL matching. + :param websocket: Match WebSocket instead of HTTP requests. A + websocket request has a ``ws`` or ``wss`` + :attr:`url_scheme`. This overrides that detection. + + .. versionadded:: 1.0 + Added ``websocket``. + + .. versionchanged:: 0.8 + ``query_args`` can be a string. + + .. versionadded:: 0.7 + Added ``query_args``. + + .. versionadded:: 0.6 + Added ``return_rule``. + """ + self.map.update() + if path_info is None: + path_info = self.path_info + if query_args is None: + query_args = self.query_args or {} + method = (method or self.default_method).upper() + + if websocket is None: + websocket = self.websocket + + domain_part = self.server_name + + if not self.map.host_matching and self.subdomain is not None: + domain_part = self.subdomain + + path_part = f"/{path_info.lstrip('/')}" if path_info else "" + + try: + result = self.map._matcher.match(domain_part, path_part, method, websocket) + except RequestPath as e: + # safe = https://url.spec.whatwg.org/#url-path-segment-string + new_path = quote(e.path_info, safe="!$&'()*+,/:;=@") + raise RequestRedirect( + self.make_redirect_url(new_path, query_args) + ) from None + except RequestAliasRedirect as e: + raise RequestRedirect( + self.make_alias_redirect_url( + f"{domain_part}|{path_part}", + e.endpoint, + e.matched_values, + method, + query_args, + ) + ) from None + except NoMatch as e: + if e.have_match_for: + raise MethodNotAllowed(valid_methods=list(e.have_match_for)) from None + + if e.websocket_mismatch: + raise WebsocketMismatch() from None + + raise NotFound() from None + else: + rule, rv = result + + if self.map.redirect_defaults: + redirect_url = self.get_default_redirect(rule, method, rv, query_args) + if redirect_url is not None: + raise RequestRedirect(redirect_url) + + if rule.redirect_to is not None: + if isinstance(rule.redirect_to, str): + + def _handle_match(match: t.Match[str]) -> str: + value = rv[match.group(1)] + return rule._converters[match.group(1)].to_url(value) + + redirect_url = _simple_rule_re.sub(_handle_match, rule.redirect_to) + else: + redirect_url = rule.redirect_to(self, **rv) + + if self.subdomain: + netloc = f"{self.subdomain}.{self.server_name}" + else: + netloc = self.server_name + + raise RequestRedirect( + urljoin( + f"{self.url_scheme or 'http'}://{netloc}{self.script_name}", + redirect_url, + ) + ) + + if return_rule: + return rule, rv + else: + return rule.endpoint, rv + + def test(self, path_info: str | None = None, method: str | None = None) -> bool: + """Test if a rule would match. Works like `match` but returns `True` + if the URL matches, or `False` if it does not exist. + + :param path_info: the path info to use for matching. Overrides the + path info specified on binding. + :param method: the HTTP method used for matching. Overrides the + method specified on binding. + """ + try: + self.match(path_info, method) + except RequestRedirect: + pass + except HTTPException: + return False + return True + + def allowed_methods(self, path_info: str | None = None) -> t.Iterable[str]: + """Returns the valid methods that match for a given path. + + .. versionadded:: 0.7 + """ + try: + self.match(path_info, method="--") + except MethodNotAllowed as e: + return e.valid_methods # type: ignore + except HTTPException: + pass + return [] + + def get_host(self, domain_part: str | None) -> str: + """Figures out the full host name for the given domain part. The + domain part is a subdomain in case host matching is disabled or + a full host name. + """ + if self.map.host_matching: + if domain_part is None: + return self.server_name + + return domain_part + + if domain_part is None: + subdomain = self.subdomain + else: + subdomain = domain_part + + if subdomain: + return f"{subdomain}.{self.server_name}" + else: + return self.server_name + + def get_default_redirect( + self, + rule: Rule, + method: str, + values: t.MutableMapping[str, t.Any], + query_args: t.Mapping[str, t.Any] | str, + ) -> str | None: + """A helper that returns the URL to redirect to if it finds one. + This is used for default redirecting only. + + :internal: + """ + assert self.map.redirect_defaults + for r in self.map._rules_by_endpoint[rule.endpoint]: + # every rule that comes after this one, including ourself + # has a lower priority for the defaults. We order the ones + # with the highest priority up for building. + if r is rule: + break + if r.provides_defaults_for(rule) and r.suitable_for(values, method): + values.update(r.defaults) # type: ignore + domain_part, path = r.build(values) # type: ignore + return self.make_redirect_url(path, query_args, domain_part=domain_part) + return None + + def encode_query_args(self, query_args: t.Mapping[str, t.Any] | str) -> str: + if not isinstance(query_args, str): + return _urlencode(query_args) + return query_args + + def make_redirect_url( + self, + path_info: str, + query_args: t.Mapping[str, t.Any] | str | None = None, + domain_part: str | None = None, + ) -> str: + """Creates a redirect URL. + + :internal: + """ + if query_args is None: + query_args = self.query_args + + if query_args: + query_str = self.encode_query_args(query_args) + else: + query_str = None + + scheme = self.url_scheme or "http" + host = self.get_host(domain_part) + path = "/".join((self.script_name.strip("/"), path_info.lstrip("/"))) + return urlunsplit((scheme, host, path, query_str, None)) + + def make_alias_redirect_url( + self, + path: str, + endpoint: t.Any, + values: t.Mapping[str, t.Any], + method: str, + query_args: t.Mapping[str, t.Any] | str, + ) -> str: + """Internally called to make an alias redirect URL.""" + url = self.build( + endpoint, values, method, append_unknown=False, force_external=True + ) + if query_args: + url += f"?{self.encode_query_args(query_args)}" + assert url != path, "detected invalid alias setting. No canonical URL found" + return url + + def _partial_build( + self, + endpoint: t.Any, + values: t.Mapping[str, t.Any], + method: str | None, + append_unknown: bool, + ) -> tuple[str, str, bool] | None: + """Helper for :meth:`build`. Returns subdomain and path for the + rule that accepts this endpoint, values and method. + + :internal: + """ + # in case the method is none, try with the default method first + if method is None: + rv = self._partial_build( + endpoint, values, self.default_method, append_unknown + ) + if rv is not None: + return rv + + # Default method did not match or a specific method is passed. + # Check all for first match with matching host. If no matching + # host is found, go with first result. + first_match = None + + for rule in self.map._rules_by_endpoint.get(endpoint, ()): + if rule.suitable_for(values, method): + build_rv = rule.build(values, append_unknown) + + if build_rv is not None: + rv = (build_rv[0], build_rv[1], rule.websocket) + if self.map.host_matching: + if rv[0] == self.server_name: + return rv + elif first_match is None: + first_match = rv + else: + return rv + + return first_match + + def build( + self, + endpoint: t.Any, + values: t.Mapping[str, t.Any] | None = None, + method: str | None = None, + force_external: bool = False, + append_unknown: bool = True, + url_scheme: str | None = None, + ) -> str: + """Building URLs works pretty much the other way round. Instead of + `match` you call `build` and pass it the endpoint and a dict of + arguments for the placeholders. + + The `build` function also accepts an argument called `force_external` + which, if you set it to `True` will force external URLs. Per default + external URLs (include the server name) will only be used if the + target URL is on a different subdomain. + + >>> m = Map([ + ... Rule('/', endpoint='index'), + ... Rule('/downloads/', endpoint='downloads/index'), + ... Rule('/downloads/', endpoint='downloads/show') + ... ]) + >>> urls = m.bind("example.com", "/") + >>> urls.build("index", {}) + '/' + >>> urls.build("downloads/show", {'id': 42}) + '/downloads/42' + >>> urls.build("downloads/show", {'id': 42}, force_external=True) + 'http://example.com/downloads/42' + + Because URLs cannot contain non ASCII data you will always get + bytes back. Non ASCII characters are urlencoded with the + charset defined on the map instance. + + Additional values are converted to strings and appended to the URL as + URL querystring parameters: + + >>> urls.build("index", {'q': 'My Searchstring'}) + '/?q=My+Searchstring' + + When processing those additional values, lists are furthermore + interpreted as multiple values (as per + :py:class:`werkzeug.datastructures.MultiDict`): + + >>> urls.build("index", {'q': ['a', 'b', 'c']}) + '/?q=a&q=b&q=c' + + Passing a ``MultiDict`` will also add multiple values: + + >>> urls.build("index", MultiDict((('p', 'z'), ('q', 'a'), ('q', 'b')))) + '/?p=z&q=a&q=b' + + If a rule does not exist when building a `BuildError` exception is + raised. + + The build method accepts an argument called `method` which allows you + to specify the method you want to have an URL built for if you have + different methods for the same endpoint specified. + + :param endpoint: the endpoint of the URL to build. + :param values: the values for the URL to build. Unhandled values are + appended to the URL as query parameters. + :param method: the HTTP method for the rule if there are different + URLs for different methods on the same endpoint. + :param force_external: enforce full canonical external URLs. If the URL + scheme is not provided, this will generate + a protocol-relative URL. + :param append_unknown: unknown parameters are appended to the generated + URL as query string argument. Disable this + if you want the builder to ignore those. + :param url_scheme: Scheme to use in place of the bound + :attr:`url_scheme`. + + .. versionchanged:: 2.0 + Added the ``url_scheme`` parameter. + + .. versionadded:: 0.6 + Added the ``append_unknown`` parameter. + """ + self.map.update() + + if values: + if isinstance(values, MultiDict): + values = { + k: (v[0] if len(v) == 1 else v) + for k, v in dict.items(values) + if len(v) != 0 + } + else: # plain dict + values = {k: v for k, v in values.items() if v is not None} + else: + values = {} + + rv = self._partial_build(endpoint, values, method, append_unknown) + if rv is None: + raise BuildError(endpoint, values, method, self) + + domain_part, path, websocket = rv + host = self.get_host(domain_part) + + if url_scheme is None: + url_scheme = self.url_scheme + + # Always build WebSocket routes with the scheme (browsers + # require full URLs). If bound to a WebSocket, ensure that HTTP + # routes are built with an HTTP scheme. + secure = url_scheme in {"https", "wss"} + + if websocket: + force_external = True + url_scheme = "wss" if secure else "ws" + elif url_scheme: + url_scheme = "https" if secure else "http" + + # shortcut this. + if not force_external and ( + (self.map.host_matching and host == self.server_name) + or (not self.map.host_matching and domain_part == self.subdomain) + ): + return f"{self.script_name.rstrip('/')}/{path.lstrip('/')}" + + scheme = f"{url_scheme}:" if url_scheme else "" + return f"{scheme}//{host}{self.script_name[:-1]}/{path.lstrip('/')}" diff --git a/lib/python3.12/site-packages/werkzeug/routing/matcher.py b/lib/python3.12/site-packages/werkzeug/routing/matcher.py new file mode 100644 index 0000000000000000000000000000000000000000..1fd00efcae4958bca9ccba1cdb957c77835a3b08 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/routing/matcher.py @@ -0,0 +1,202 @@ +from __future__ import annotations + +import re +import typing as t +from dataclasses import dataclass +from dataclasses import field + +from .converters import ValidationError +from .exceptions import NoMatch +from .exceptions import RequestAliasRedirect +from .exceptions import RequestPath +from .rules import Rule +from .rules import RulePart + + +class SlashRequired(Exception): + pass + + +@dataclass +class State: + """A representation of a rule state. + + This includes the *rules* that correspond to the state and the + possible *static* and *dynamic* transitions to the next state. + """ + + dynamic: list[tuple[RulePart, State]] = field(default_factory=list) + rules: list[Rule] = field(default_factory=list) + static: dict[str, State] = field(default_factory=dict) + + +class StateMachineMatcher: + def __init__(self, merge_slashes: bool) -> None: + self._root = State() + self.merge_slashes = merge_slashes + + def add(self, rule: Rule) -> None: + state = self._root + for part in rule._parts: + if part.static: + state.static.setdefault(part.content, State()) + state = state.static[part.content] + else: + for test_part, new_state in state.dynamic: + if test_part == part: + state = new_state + break + else: + new_state = State() + state.dynamic.append((part, new_state)) + state = new_state + state.rules.append(rule) + + def update(self) -> None: + # For every state the dynamic transitions should be sorted by + # the weight of the transition + state = self._root + + def _update_state(state: State) -> None: + state.dynamic.sort(key=lambda entry: entry[0].weight) + for new_state in state.static.values(): + _update_state(new_state) + for _, new_state in state.dynamic: + _update_state(new_state) + + _update_state(state) + + def match( + self, domain: str, path: str, method: str, websocket: bool + ) -> tuple[Rule, t.MutableMapping[str, t.Any]]: + # To match to a rule we need to start at the root state and + # try to follow the transitions until we find a match, or find + # there is no transition to follow. + + have_match_for = set() + websocket_mismatch = False + + def _match( + state: State, parts: list[str], values: list[str] + ) -> tuple[Rule, list[str]] | None: + # This function is meant to be called recursively, and will attempt + # to match the head part to the state's transitions. + nonlocal have_match_for, websocket_mismatch + + # The base case is when all parts have been matched via + # transitions. Hence if there is a rule with methods & + # websocket that work return it and the dynamic values + # extracted. + if parts == []: + for rule in state.rules: + if rule.methods is not None and method not in rule.methods: + have_match_for.update(rule.methods) + elif rule.websocket != websocket: + websocket_mismatch = True + else: + return rule, values + + # Test if there is a match with this path with a + # trailing slash, if so raise an exception to report + # that matching is possible with an additional slash + if "" in state.static: + for rule in state.static[""].rules: + if websocket == rule.websocket and ( + rule.methods is None or method in rule.methods + ): + if rule.strict_slashes: + raise SlashRequired() + else: + return rule, values + return None + + part = parts[0] + # To match this part try the static transitions first + if part in state.static: + rv = _match(state.static[part], parts[1:], values) + if rv is not None: + return rv + # No match via the static transitions, so try the dynamic + # ones. + for test_part, new_state in state.dynamic: + target = part + remaining = parts[1:] + # A final part indicates a transition that always + # consumes the remaining parts i.e. transitions to a + # final state. + if test_part.final: + target = "/".join(parts) + remaining = [] + match = re.compile(test_part.content).match(target) + if match is not None: + if test_part.suffixed: + # If a part_isolating=False part has a slash suffix, remove the + # suffix from the match and check for the slash redirect next. + suffix = match.groups()[-1] + if suffix == "/": + remaining = [""] + + converter_groups = sorted( + match.groupdict().items(), key=lambda entry: entry[0] + ) + groups = [ + value + for key, value in converter_groups + if key[:11] == "__werkzeug_" + ] + rv = _match(new_state, remaining, values + groups) + if rv is not None: + return rv + + # If there is no match and the only part left is a + # trailing slash ("") consider rules that aren't + # strict-slashes as these should match if there is a final + # slash part. + if parts == [""]: + for rule in state.rules: + if rule.strict_slashes: + continue + if rule.methods is not None and method not in rule.methods: + have_match_for.update(rule.methods) + elif rule.websocket != websocket: + websocket_mismatch = True + else: + return rule, values + + return None + + try: + rv = _match(self._root, [domain, *path.split("/")], []) + except SlashRequired: + raise RequestPath(f"{path}/") from None + + if self.merge_slashes and rv is None: + # Try to match again, but with slashes merged + path = re.sub("/{2,}?", "/", path) + try: + rv = _match(self._root, [domain, *path.split("/")], []) + except SlashRequired: + raise RequestPath(f"{path}/") from None + if rv is None or rv[0].merge_slashes is False: + raise NoMatch(have_match_for, websocket_mismatch) + else: + raise RequestPath(f"{path}") + elif rv is not None: + rule, values = rv + + result = {} + for name, value in zip(rule._converters.keys(), values): + try: + value = rule._converters[name].to_python(value) + except ValidationError: + raise NoMatch(have_match_for, websocket_mismatch) from None + result[str(name)] = value + if rule.defaults: + result.update(rule.defaults) + + if rule.alias and rule.map.redirect_defaults: + raise RequestAliasRedirect(result, rule.endpoint) + + return rule, result + + raise NoMatch(have_match_for, websocket_mismatch) diff --git a/lib/python3.12/site-packages/werkzeug/routing/rules.py b/lib/python3.12/site-packages/werkzeug/routing/rules.py new file mode 100644 index 0000000000000000000000000000000000000000..9e483a54b284e6f7dcc0478a165634cc4794391d --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/routing/rules.py @@ -0,0 +1,928 @@ +from __future__ import annotations + +import ast +import re +import typing as t +from dataclasses import dataclass +from string import Template +from types import CodeType +from urllib.parse import quote + +from ..datastructures import iter_multi_items +from ..urls import _urlencode +from .converters import ValidationError + +if t.TYPE_CHECKING: + from .converters import BaseConverter + from .map import Map + + +class Weighting(t.NamedTuple): + number_static_weights: int + static_weights: list[tuple[int, int]] + number_argument_weights: int + argument_weights: list[int] + + +@dataclass +class RulePart: + """A part of a rule. + + Rules can be represented by parts as delimited by `/` with + instances of this class representing those parts. The *content* is + either the raw content if *static* or a regex string to match + against. The *weight* can be used to order parts when matching. + + """ + + content: str + final: bool + static: bool + suffixed: bool + weight: Weighting + + +_part_re = re.compile( + r""" + (?: + (?P/) # a slash + | + (?P[^[a-zA-Z_][a-zA-Z0-9_]*) # converter name + (?:\((?P.*?)\))? # converter arguments + : # variable delimiter + )? + (?P[a-zA-Z_][a-zA-Z0-9_]*) # variable name + > + ) + ) + """, + re.VERBOSE, +) + +_simple_rule_re = re.compile(r"<([^>]+)>") +_converter_args_re = re.compile( + r""" + \s* + ((?P\w+)\s*=\s*)? + (?P + True|False| + \d+.\d+| + \d+.| + \d+| + [\w\d_.]+| + [urUR]?(?P"[^"]*?"|'[^']*') + )\s*, + """, + re.VERBOSE, +) + + +_PYTHON_CONSTANTS = {"None": None, "True": True, "False": False} + + +def _find(value: str, target: str, pos: int) -> int: + """Find the *target* in *value* after *pos*. + + Returns the *value* length if *target* isn't found. + """ + try: + return value.index(target, pos) + except ValueError: + return len(value) + + +def _pythonize(value: str) -> None | bool | int | float | str: + if value in _PYTHON_CONSTANTS: + return _PYTHON_CONSTANTS[value] + for convert in int, float: + try: + return convert(value) + except ValueError: + pass + if value[:1] == value[-1:] and value[0] in "\"'": + value = value[1:-1] + return str(value) + + +def parse_converter_args(argstr: str) -> tuple[tuple[t.Any, ...], dict[str, t.Any]]: + argstr += "," + args = [] + kwargs = {} + position = 0 + + for item in _converter_args_re.finditer(argstr): + if item.start() != position: + raise ValueError( + f"Cannot parse converter argument '{argstr[position : item.start()]}'" + ) + + value = item.group("stringval") + if value is None: + value = item.group("value") + value = _pythonize(value) + if not item.group("name"): + args.append(value) + else: + name = item.group("name") + kwargs[name] = value + position = item.end() + + return tuple(args), kwargs + + +class RuleFactory: + """As soon as you have more complex URL setups it's a good idea to use rule + factories to avoid repetitive tasks. Some of them are builtin, others can + be added by subclassing `RuleFactory` and overriding `get_rules`. + """ + + def get_rules(self, map: Map) -> t.Iterable[Rule]: + """Subclasses of `RuleFactory` have to override this method and return + an iterable of rules.""" + raise NotImplementedError() + + +class Subdomain(RuleFactory): + """All URLs provided by this factory have the subdomain set to a + specific domain. For example if you want to use the subdomain for + the current language this can be a good setup:: + + url_map = Map([ + Rule('/', endpoint='#select_language'), + Subdomain('', [ + Rule('/', endpoint='index'), + Rule('/about', endpoint='about'), + Rule('/help', endpoint='help') + ]) + ]) + + All the rules except for the ``'#select_language'`` endpoint will now + listen on a two letter long subdomain that holds the language code + for the current request. + """ + + def __init__(self, subdomain: str, rules: t.Iterable[RuleFactory]) -> None: + self.subdomain = subdomain + self.rules = rules + + def get_rules(self, map: Map) -> t.Iterator[Rule]: + for rulefactory in self.rules: + for rule in rulefactory.get_rules(map): + rule = rule.empty() + rule.subdomain = self.subdomain + yield rule + + +class Submount(RuleFactory): + """Like `Subdomain` but prefixes the URL rule with a given string:: + + url_map = Map([ + Rule('/', endpoint='index'), + Submount('/blog', [ + Rule('/', endpoint='blog/index'), + Rule('/entry/', endpoint='blog/show') + ]) + ]) + + Now the rule ``'blog/show'`` matches ``/blog/entry/``. + """ + + def __init__(self, path: str, rules: t.Iterable[RuleFactory]) -> None: + self.path = path.rstrip("/") + self.rules = rules + + def get_rules(self, map: Map) -> t.Iterator[Rule]: + for rulefactory in self.rules: + for rule in rulefactory.get_rules(map): + rule = rule.empty() + rule.rule = self.path + rule.rule + yield rule + + +class EndpointPrefix(RuleFactory): + """Prefixes all endpoints (which must be strings for this factory) with + another string. This can be useful for sub applications:: + + url_map = Map([ + Rule('/', endpoint='index'), + EndpointPrefix('blog/', [Submount('/blog', [ + Rule('/', endpoint='index'), + Rule('/entry/', endpoint='show') + ])]) + ]) + """ + + def __init__(self, prefix: str, rules: t.Iterable[RuleFactory]) -> None: + self.prefix = prefix + self.rules = rules + + def get_rules(self, map: Map) -> t.Iterator[Rule]: + for rulefactory in self.rules: + for rule in rulefactory.get_rules(map): + rule = rule.empty() + rule.endpoint = self.prefix + rule.endpoint + yield rule + + +class RuleTemplate: + """Returns copies of the rules wrapped and expands string templates in + the endpoint, rule, defaults or subdomain sections. + + Here a small example for such a rule template:: + + from werkzeug.routing import Map, Rule, RuleTemplate + + resource = RuleTemplate([ + Rule('/$name/', endpoint='$name.list'), + Rule('/$name/', endpoint='$name.show') + ]) + + url_map = Map([resource(name='user'), resource(name='page')]) + + When a rule template is called the keyword arguments are used to + replace the placeholders in all the string parameters. + """ + + def __init__(self, rules: t.Iterable[Rule]) -> None: + self.rules = list(rules) + + def __call__(self, *args: t.Any, **kwargs: t.Any) -> RuleTemplateFactory: + return RuleTemplateFactory(self.rules, dict(*args, **kwargs)) + + +class RuleTemplateFactory(RuleFactory): + """A factory that fills in template variables into rules. Used by + `RuleTemplate` internally. + + :internal: + """ + + def __init__( + self, rules: t.Iterable[RuleFactory], context: dict[str, t.Any] + ) -> None: + self.rules = rules + self.context = context + + def get_rules(self, map: Map) -> t.Iterator[Rule]: + for rulefactory in self.rules: + for rule in rulefactory.get_rules(map): + new_defaults = subdomain = None + if rule.defaults: + new_defaults = {} + for key, value in rule.defaults.items(): + if isinstance(value, str): + value = Template(value).substitute(self.context) + new_defaults[key] = value + if rule.subdomain is not None: + subdomain = Template(rule.subdomain).substitute(self.context) + new_endpoint = rule.endpoint + if isinstance(new_endpoint, str): + new_endpoint = Template(new_endpoint).substitute(self.context) + yield Rule( + Template(rule.rule).substitute(self.context), + new_defaults, + subdomain, + rule.methods, + rule.build_only, + new_endpoint, + rule.strict_slashes, + ) + + +_ASTT = t.TypeVar("_ASTT", bound=ast.AST) + + +def _prefix_names(src: str, expected_type: type[_ASTT]) -> _ASTT: + """ast parse and prefix names with `.` to avoid collision with user vars""" + tree: ast.AST = ast.parse(src).body[0] + if isinstance(tree, ast.Expr): + tree = tree.value + if not isinstance(tree, expected_type): + raise TypeError( + f"AST node is of type {type(tree).__name__}, not {expected_type.__name__}" + ) + for node in ast.walk(tree): + if isinstance(node, ast.Name): + node.id = f".{node.id}" + return tree + + +_CALL_CONVERTER_CODE_FMT = "self._converters[{elem!r}].to_url()" +_IF_KWARGS_URL_ENCODE_CODE = """\ +if kwargs: + params = self._encode_query_vars(kwargs) + q = "?" if params else "" +else: + q = params = "" +""" +_IF_KWARGS_URL_ENCODE_AST = _prefix_names(_IF_KWARGS_URL_ENCODE_CODE, ast.If) +_URL_ENCODE_AST_NAMES = ( + _prefix_names("q", ast.Name), + _prefix_names("params", ast.Name), +) + + +class Rule(RuleFactory): + """A Rule represents one URL pattern. There are some options for `Rule` + that change the way it behaves and are passed to the `Rule` constructor. + Note that besides the rule-string all arguments *must* be keyword arguments + in order to not break the application on Werkzeug upgrades. + + `string` + Rule strings basically are just normal URL paths with placeholders in + the format ```` where the converter and the + arguments are optional. If no converter is defined the `default` + converter is used which means `string` in the normal configuration. + + URL rules that end with a slash are branch URLs, others are leaves. + If you have `strict_slashes` enabled (which is the default), all + branch URLs that are matched without a trailing slash will trigger a + redirect to the same URL with the missing slash appended. + + The converters are defined on the `Map`. + + `endpoint` + The endpoint for this rule. This can be anything. A reference to a + function, a string, a number etc. The preferred way is using a string + because the endpoint is used for URL generation. + + `defaults` + An optional dict with defaults for other rules with the same endpoint. + This is a bit tricky but useful if you want to have unique URLs:: + + url_map = Map([ + Rule('/all/', defaults={'page': 1}, endpoint='all_entries'), + Rule('/all/page/', endpoint='all_entries') + ]) + + If a user now visits ``http://example.com/all/page/1`` they will be + redirected to ``http://example.com/all/``. If `redirect_defaults` is + disabled on the `Map` instance this will only affect the URL + generation. + + `subdomain` + The subdomain rule string for this rule. If not specified the rule + only matches for the `default_subdomain` of the map. If the map is + not bound to a subdomain this feature is disabled. + + Can be useful if you want to have user profiles on different subdomains + and all subdomains are forwarded to your application:: + + url_map = Map([ + Rule('/', subdomain='', endpoint='user/homepage'), + Rule('/stats', subdomain='', endpoint='user/stats') + ]) + + `methods` + A sequence of http methods this rule applies to. If not specified, all + methods are allowed. For example this can be useful if you want different + endpoints for `POST` and `GET`. If methods are defined and the path + matches but the method matched against is not in this list or in the + list of another rule for that path the error raised is of the type + `MethodNotAllowed` rather than `NotFound`. If `GET` is present in the + list of methods and `HEAD` is not, `HEAD` is added automatically. + + `strict_slashes` + Override the `Map` setting for `strict_slashes` only for this rule. If + not specified the `Map` setting is used. + + `merge_slashes` + Override :attr:`Map.merge_slashes` for this rule. + + `build_only` + Set this to True and the rule will never match but will create a URL + that can be build. This is useful if you have resources on a subdomain + or folder that are not handled by the WSGI application (like static data) + + `redirect_to` + If given this must be either a string or callable. In case of a + callable it's called with the url adapter that triggered the match and + the values of the URL as keyword arguments and has to return the target + for the redirect, otherwise it has to be a string with placeholders in + rule syntax:: + + def foo_with_slug(adapter, id): + # ask the database for the slug for the old id. this of + # course has nothing to do with werkzeug. + return f'foo/{Foo.get_slug_for_id(id)}' + + url_map = Map([ + Rule('/foo/', endpoint='foo'), + Rule('/some/old/url/', redirect_to='foo/'), + Rule('/other/old/url/', redirect_to=foo_with_slug) + ]) + + When the rule is matched the routing system will raise a + `RequestRedirect` exception with the target for the redirect. + + Keep in mind that the URL will be joined against the URL root of the + script so don't use a leading slash on the target URL unless you + really mean root of that domain. + + `alias` + If enabled this rule serves as an alias for another rule with the same + endpoint and arguments. + + `host` + If provided and the URL map has host matching enabled this can be + used to provide a match rule for the whole host. This also means + that the subdomain feature is disabled. + + `websocket` + If ``True``, this rule is only matches for WebSocket (``ws://``, + ``wss://``) requests. By default, rules will only match for HTTP + requests. + + .. versionchanged:: 2.1 + Percent-encoded newlines (``%0a``), which are decoded by WSGI + servers, are considered when routing instead of terminating the + match early. + + .. versionadded:: 1.0 + Added ``websocket``. + + .. versionadded:: 1.0 + Added ``merge_slashes``. + + .. versionadded:: 0.7 + Added ``alias`` and ``host``. + + .. versionchanged:: 0.6.1 + ``HEAD`` is added to ``methods`` if ``GET`` is present. + """ + + def __init__( + self, + string: str, + defaults: t.Mapping[str, t.Any] | None = None, + subdomain: str | None = None, + methods: t.Iterable[str] | None = None, + build_only: bool = False, + endpoint: t.Any | None = None, + strict_slashes: bool | None = None, + merge_slashes: bool | None = None, + redirect_to: str | t.Callable[..., str] | None = None, + alias: bool = False, + host: str | None = None, + websocket: bool = False, + ) -> None: + if not string.startswith("/"): + raise ValueError(f"URL rule '{string}' must start with a slash.") + + self.rule = string + self.is_leaf = not string.endswith("/") + self.is_branch = string.endswith("/") + + self.map: Map = None # type: ignore + self.strict_slashes = strict_slashes + self.merge_slashes = merge_slashes + self.subdomain = subdomain + self.host = host + self.defaults = defaults + self.build_only = build_only + self.alias = alias + self.websocket = websocket + + if methods is not None: + if isinstance(methods, str): + raise TypeError("'methods' should be a list of strings.") + + methods = {x.upper() for x in methods} + + if "HEAD" not in methods and "GET" in methods: + methods.add("HEAD") + + if websocket and methods - {"GET", "HEAD", "OPTIONS"}: + raise ValueError( + "WebSocket rules can only use 'GET', 'HEAD', and 'OPTIONS' methods." + ) + + self.methods = methods + self.endpoint: t.Any = endpoint + self.redirect_to = redirect_to + + if defaults: + self.arguments = set(map(str, defaults)) + else: + self.arguments = set() + + self._converters: dict[str, BaseConverter] = {} + self._trace: list[tuple[bool, str]] = [] + self._parts: list[RulePart] = [] + + def empty(self) -> Rule: + """ + Return an unbound copy of this rule. + + This can be useful if want to reuse an already bound URL for another + map. See ``get_empty_kwargs`` to override what keyword arguments are + provided to the new copy. + """ + return type(self)(self.rule, **self.get_empty_kwargs()) + + def get_empty_kwargs(self) -> t.Mapping[str, t.Any]: + """ + Provides kwargs for instantiating empty copy with empty() + + Use this method to provide custom keyword arguments to the subclass of + ``Rule`` when calling ``some_rule.empty()``. Helpful when the subclass + has custom keyword arguments that are needed at instantiation. + + Must return a ``dict`` that will be provided as kwargs to the new + instance of ``Rule``, following the initial ``self.rule`` value which + is always provided as the first, required positional argument. + """ + defaults = None + if self.defaults: + defaults = dict(self.defaults) + return dict( + defaults=defaults, + subdomain=self.subdomain, + methods=self.methods, + build_only=self.build_only, + endpoint=self.endpoint, + strict_slashes=self.strict_slashes, + redirect_to=self.redirect_to, + alias=self.alias, + host=self.host, + ) + + def get_rules(self, map: Map) -> t.Iterator[Rule]: + yield self + + def refresh(self) -> None: + """Rebinds and refreshes the URL. Call this if you modified the + rule in place. + + :internal: + """ + self.bind(self.map, rebind=True) + + def bind(self, map: Map, rebind: bool = False) -> None: + """Bind the url to a map and create a regular expression based on + the information from the rule itself and the defaults from the map. + + :internal: + """ + if self.map is not None and not rebind: + raise RuntimeError(f"url rule {self!r} already bound to map {self.map!r}") + self.map = map + if self.strict_slashes is None: + self.strict_slashes = map.strict_slashes + if self.merge_slashes is None: + self.merge_slashes = map.merge_slashes + if self.subdomain is None: + self.subdomain = map.default_subdomain + self.compile() + + def get_converter( + self, + variable_name: str, + converter_name: str, + args: tuple[t.Any, ...], + kwargs: t.Mapping[str, t.Any], + ) -> BaseConverter: + """Looks up the converter for the given parameter. + + .. versionadded:: 0.9 + """ + if converter_name not in self.map.converters: + raise LookupError(f"the converter {converter_name!r} does not exist") + return self.map.converters[converter_name](self.map, *args, **kwargs) + + def _encode_query_vars(self, query_vars: t.Mapping[str, t.Any]) -> str: + items: t.Iterable[tuple[str, str]] = iter_multi_items(query_vars) + + if self.map.sort_parameters: + items = sorted(items, key=self.map.sort_key) + + return _urlencode(items) + + def _parse_rule(self, rule: str) -> t.Iterable[RulePart]: + content = "" + static = True + argument_weights = [] + static_weights: list[tuple[int, int]] = [] + final = False + convertor_number = 0 + + pos = 0 + while pos < len(rule): + match = _part_re.match(rule, pos) + if match is None: + raise ValueError(f"malformed url rule: {rule!r}") + + data = match.groupdict() + if data["static"] is not None: + static_weights.append((len(static_weights), -len(data["static"]))) + self._trace.append((False, data["static"])) + content += data["static"] if static else re.escape(data["static"]) + + if data["variable"] is not None: + if static: + # Switching content to represent regex, hence the need to escape + content = re.escape(content) + static = False + c_args, c_kwargs = parse_converter_args(data["arguments"] or "") + convobj = self.get_converter( + data["variable"], data["converter"] or "default", c_args, c_kwargs + ) + self._converters[data["variable"]] = convobj + self.arguments.add(data["variable"]) + if not convobj.part_isolating: + final = True + content += f"(?P<__werkzeug_{convertor_number}>{convobj.regex})" + convertor_number += 1 + argument_weights.append(convobj.weight) + self._trace.append((True, data["variable"])) + + if data["slash"] is not None: + self._trace.append((False, "/")) + if final: + content += "/" + else: + if not static: + content += r"\Z" + weight = Weighting( + -len(static_weights), + static_weights, + -len(argument_weights), + argument_weights, + ) + yield RulePart( + content=content, + final=final, + static=static, + suffixed=False, + weight=weight, + ) + content = "" + static = True + argument_weights = [] + static_weights = [] + final = False + convertor_number = 0 + + pos = match.end() + + suffixed = False + if final and content[-1] == "/": + # If a converter is part_isolating=False (matches slashes) and ends with a + # slash, augment the regex to support slash redirects. + suffixed = True + content = content[:-1] + "(? None: + """Compiles the regular expression and stores it.""" + assert self.map is not None, "rule not bound" + + if self.map.host_matching: + domain_rule = self.host or "" + else: + domain_rule = self.subdomain or "" + self._parts = [] + self._trace = [] + self._converters = {} + if domain_rule == "": + self._parts = [ + RulePart( + content="", + final=False, + static=True, + suffixed=False, + weight=Weighting(0, [], 0, []), + ) + ] + else: + self._parts.extend(self._parse_rule(domain_rule)) + self._trace.append((False, "|")) + rule = self.rule + if self.merge_slashes: + rule = re.sub("/{2,}?", "/", self.rule) + self._parts.extend(self._parse_rule(rule)) + + self._build: t.Callable[..., tuple[str, str]] + self._build = self._compile_builder(False).__get__(self, None) + self._build_unknown: t.Callable[..., tuple[str, str]] + self._build_unknown = self._compile_builder(True).__get__(self, None) + + @staticmethod + def _get_func_code(code: CodeType, name: str) -> t.Callable[..., tuple[str, str]]: + globs: dict[str, t.Any] = {} + locs: dict[str, t.Any] = {} + exec(code, globs, locs) + return locs[name] # type: ignore + + def _compile_builder( + self, append_unknown: bool = True + ) -> t.Callable[..., tuple[str, str]]: + defaults = self.defaults or {} + dom_ops: list[tuple[bool, str]] = [] + url_ops: list[tuple[bool, str]] = [] + + opl = dom_ops + for is_dynamic, data in self._trace: + if data == "|" and opl is dom_ops: + opl = url_ops + continue + # this seems like a silly case to ever come up but: + # if a default is given for a value that appears in the rule, + # resolve it to a constant ahead of time + if is_dynamic and data in defaults: + data = self._converters[data].to_url(defaults[data]) + opl.append((False, data)) + elif not is_dynamic: + # safe = https://url.spec.whatwg.org/#url-path-segment-string + opl.append((False, quote(data, safe="!$&'()*+,/:;=@"))) + else: + opl.append((True, data)) + + def _convert(elem: str) -> ast.Call: + ret = _prefix_names(_CALL_CONVERTER_CODE_FMT.format(elem=elem), ast.Call) + ret.args = [ast.Name(elem, ast.Load())] + return ret + + def _parts(ops: list[tuple[bool, str]]) -> list[ast.expr]: + parts: list[ast.expr] = [ + _convert(elem) if is_dynamic else ast.Constant(elem) + for is_dynamic, elem in ops + ] + parts = parts or [ast.Constant("")] + # constant fold + ret = [parts[0]] + for p in parts[1:]: + if isinstance(p, ast.Constant) and isinstance(ret[-1], ast.Constant): + ret[-1] = ast.Constant(ret[-1].value + p.value) # type: ignore[operator] + else: + ret.append(p) + return ret + + dom_parts = _parts(dom_ops) + url_parts = _parts(url_ops) + body: list[ast.stmt] + if not append_unknown: + body = [] + else: + body = [_IF_KWARGS_URL_ENCODE_AST] + url_parts.extend(_URL_ENCODE_AST_NAMES) + + def _join(parts: list[ast.expr]) -> ast.expr: + if len(parts) == 1: # shortcut + return parts[0] + return ast.JoinedStr(parts) + + body.append( + ast.Return(ast.Tuple([_join(dom_parts), _join(url_parts)], ast.Load())) + ) + + pargs = [ + elem + for is_dynamic, elem in dom_ops + url_ops + if is_dynamic and elem not in defaults + ] + kargs = [str(k) for k in defaults] + + func_ast = _prefix_names("def _(): pass", ast.FunctionDef) + func_ast.name = f"" + func_ast.args.args.append(ast.arg(".self", None)) + for arg in pargs + kargs: + func_ast.args.args.append(ast.arg(arg, None)) + func_ast.args.kwarg = ast.arg(".kwargs", None) + for _ in kargs: + func_ast.args.defaults.append(ast.Constant("")) + func_ast.body = body + + # Use `ast.parse` instead of `ast.Module` for better portability, since the + # signature of `ast.Module` can change. + module = ast.parse("") + module.body = [func_ast] + + # mark everything as on line 1, offset 0 + # less error-prone than `ast.fix_missing_locations` + # bad line numbers cause an assert to fail in debug builds + for node in ast.walk(module): + if "lineno" in node._attributes: + node.lineno = 1 # type: ignore[attr-defined] + if "end_lineno" in node._attributes: + node.end_lineno = node.lineno # type: ignore[attr-defined] + if "col_offset" in node._attributes: + node.col_offset = 0 # type: ignore[attr-defined] + if "end_col_offset" in node._attributes: + node.end_col_offset = node.col_offset # type: ignore[attr-defined] + + code = compile(module, "", "exec") + return self._get_func_code(code, func_ast.name) + + def build( + self, values: t.Mapping[str, t.Any], append_unknown: bool = True + ) -> tuple[str, str] | None: + """Assembles the relative url for that rule and the subdomain. + If building doesn't work for some reasons `None` is returned. + + :internal: + """ + try: + if append_unknown: + return self._build_unknown(**values) + else: + return self._build(**values) + except ValidationError: + return None + + def provides_defaults_for(self, rule: Rule) -> bool: + """Check if this rule has defaults for a given rule. + + :internal: + """ + return bool( + not self.build_only + and self.defaults + and self.endpoint == rule.endpoint + and self != rule + and self.arguments == rule.arguments + ) + + def suitable_for( + self, values: t.Mapping[str, t.Any], method: str | None = None + ) -> bool: + """Check if the dict of values has enough data for url generation. + + :internal: + """ + # if a method was given explicitly and that method is not supported + # by this rule, this rule is not suitable. + if ( + method is not None + and self.methods is not None + and method not in self.methods + ): + return False + + defaults = self.defaults or () + + # all arguments required must be either in the defaults dict or + # the value dictionary otherwise it's not suitable + for key in self.arguments: + if key not in defaults and key not in values: + return False + + # in case defaults are given we ensure that either the value was + # skipped or the value is the same as the default value. + if defaults: + for key, value in defaults.items(): + if key in values and value != values[key]: + return False + + return True + + def build_compare_key(self) -> tuple[int, int, int]: + """The build compare key for sorting. + + :internal: + """ + return (1 if self.alias else 0, -len(self.arguments), -len(self.defaults or ())) + + def __eq__(self, other: object) -> bool: + return isinstance(other, type(self)) and self._trace == other._trace + + __hash__ = None # type: ignore + + def __str__(self) -> str: + return self.rule + + def __repr__(self) -> str: + if self.map is None: + return f"<{type(self).__name__} (unbound)>" + parts = [] + for is_dynamic, data in self._trace: + if is_dynamic: + parts.append(f"<{data}>") + else: + parts.append(data) + parts_str = "".join(parts).lstrip("|") + methods = f" ({', '.join(self.methods)})" if self.methods is not None else "" + return f"<{type(self).__name__} {parts_str!r}{methods} -> {self.endpoint}>" diff --git a/lib/python3.12/site-packages/werkzeug/sansio/__init__.py b/lib/python3.12/site-packages/werkzeug/sansio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..27d4534d048b09ffef1f0078ca88ffe4858f8939 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/http.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/http.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2d4d35e7c470364e366c2e02240c19907b70e69 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/http.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/multipart.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/multipart.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ff7ea89050a235ff32a53a070c84a861a14cce24 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/multipart.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/request.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/request.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..25128f569c613f942a9ae1621018e12dac159fbe Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/request.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/response.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/response.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ed39e2cebce81629336c35dd854d53f52b9d5bc0 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/response.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/utils.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f80b6cd1d9b2dd967bc7905e07340a5290152f9 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/sansio/__pycache__/utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/sansio/http.py b/lib/python3.12/site-packages/werkzeug/sansio/http.py new file mode 100644 index 0000000000000000000000000000000000000000..c68a57ffb06fdbcb4986be0f65d7eac711503d7e --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/sansio/http.py @@ -0,0 +1,170 @@ +from __future__ import annotations + +import re +import typing as t +from datetime import datetime + +from .._internal import _dt_as_utc +from ..http import generate_etag +from ..http import parse_date +from ..http import parse_etags +from ..http import parse_if_range_header +from ..http import unquote_etag + +_etag_re = re.compile(r'([Ww]/)?(?:"(.*?)"|(.*?))(?:\s*,\s*|$)') + + +def is_resource_modified( + http_range: str | None = None, + http_if_range: str | None = None, + http_if_modified_since: str | None = None, + http_if_none_match: str | None = None, + http_if_match: str | None = None, + etag: str | None = None, + data: bytes | None = None, + last_modified: datetime | str | None = None, + ignore_if_range: bool = True, +) -> bool: + """Convenience method for conditional requests. + :param http_range: Range HTTP header + :param http_if_range: If-Range HTTP header + :param http_if_modified_since: If-Modified-Since HTTP header + :param http_if_none_match: If-None-Match HTTP header + :param http_if_match: If-Match HTTP header + :param etag: the etag for the response for comparison. + :param data: or alternatively the data of the response to automatically + generate an etag using :func:`generate_etag`. + :param last_modified: an optional date of the last modification. + :param ignore_if_range: If `False`, `If-Range` header will be taken into + account. + :return: `True` if the resource was modified, otherwise `False`. + + .. versionadded:: 2.2 + """ + if etag is None and data is not None: + etag = generate_etag(data) + elif data is not None: + raise TypeError("both data and etag given") + + unmodified = False + if isinstance(last_modified, str): + last_modified = parse_date(last_modified) + + # HTTP doesn't use microsecond, remove it to avoid false positive + # comparisons. Mark naive datetimes as UTC. + if last_modified is not None: + last_modified = _dt_as_utc(last_modified.replace(microsecond=0)) + + if_range = None + if not ignore_if_range and http_range is not None: + # https://tools.ietf.org/html/rfc7233#section-3.2 + # A server MUST ignore an If-Range header field received in a request + # that does not contain a Range header field. + if_range = parse_if_range_header(http_if_range) + + if if_range is not None and if_range.date is not None: + modified_since: datetime | None = if_range.date + else: + modified_since = parse_date(http_if_modified_since) + + if modified_since and last_modified and last_modified <= modified_since: + unmodified = True + + if etag: + etag, _ = unquote_etag(etag) + + if if_range is not None and if_range.etag is not None: + unmodified = parse_etags(if_range.etag).contains(etag) + else: + if_none_match = parse_etags(http_if_none_match) + if if_none_match: + # https://tools.ietf.org/html/rfc7232#section-3.2 + # "A recipient MUST use the weak comparison function when comparing + # entity-tags for If-None-Match" + unmodified = if_none_match.contains_weak(etag) + + # https://tools.ietf.org/html/rfc7232#section-3.1 + # "Origin server MUST use the strong comparison function when + # comparing entity-tags for If-Match" + if_match = parse_etags(http_if_match) + if if_match: + unmodified = not if_match.is_strong(etag) + + return not unmodified + + +_cookie_re = re.compile( + r""" + ([^=;]*) + (?:\s*=\s* + ( + "(?:[^\\"]|\\.)*" + | + .*? + ) + )? + \s*;\s* + """, + flags=re.ASCII | re.VERBOSE, +) +_cookie_unslash_re = re.compile(rb"\\([0-3][0-7]{2}|.)") + + +def _cookie_unslash_replace(m: t.Match[bytes]) -> bytes: + v = m.group(1) + + if len(v) == 1: + return v + + return int(v, 8).to_bytes(1, "big") + + +def parse_cookie( + cookie: str | None = None, + cls: type[ds.MultiDict[str, str]] | None = None, +) -> ds.MultiDict[str, str]: + """Parse a cookie from a string. + + The same key can be provided multiple times, the values are stored + in-order. The default :class:`MultiDict` will have the first value + first, and all values can be retrieved with + :meth:`MultiDict.getlist`. + + :param cookie: The cookie header as a string. + :param cls: A dict-like class to store the parsed cookies in. + Defaults to :class:`MultiDict`. + + .. versionchanged:: 3.0 + Passing bytes, and the ``charset`` and ``errors`` parameters, were removed. + + .. versionadded:: 2.2 + """ + if cls is None: + cls = t.cast("type[ds.MultiDict[str, str]]", ds.MultiDict) + + if not cookie: + return cls() + + cookie = f"{cookie};" + out = [] + + for ck, cv in _cookie_re.findall(cookie): + ck = ck.strip() + cv = cv.strip() + + if not ck: + continue + + if len(cv) >= 2 and cv[0] == cv[-1] == '"': + # Work with bytes here, since a UTF-8 character could be multiple bytes. + cv = _cookie_unslash_re.sub( + _cookie_unslash_replace, cv[1:-1].encode() + ).decode(errors="replace") + + out.append((ck, cv)) + + return cls(out) + + +# circular dependencies +from .. import datastructures as ds # noqa: E402 diff --git a/lib/python3.12/site-packages/werkzeug/sansio/multipart.py b/lib/python3.12/site-packages/werkzeug/sansio/multipart.py new file mode 100644 index 0000000000000000000000000000000000000000..5a3844ebbe80d9043e9c57a210ce1b5b1ddcb392 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/sansio/multipart.py @@ -0,0 +1,329 @@ +from __future__ import annotations + +import re +import typing as t +from dataclasses import dataclass +from enum import auto +from enum import Enum + +from ..datastructures import Headers +from ..exceptions import RequestEntityTooLarge +from ..http import parse_options_header + + +class Event: + pass + + +@dataclass(frozen=True) +class Preamble(Event): + data: bytes + + +@dataclass(frozen=True) +class Field(Event): + name: str + headers: Headers + + +@dataclass(frozen=True) +class File(Event): + name: str + filename: str + headers: Headers + + +@dataclass(frozen=True) +class Data(Event): + data: bytes + more_data: bool + + +@dataclass(frozen=True) +class Epilogue(Event): + data: bytes + + +class NeedData(Event): + pass + + +NEED_DATA = NeedData() + + +class State(Enum): + PREAMBLE = auto() + PART = auto() + DATA = auto() + DATA_START = auto() + EPILOGUE = auto() + COMPLETE = auto() + + +# Multipart line breaks MUST be CRLF (\r\n) by RFC-7578, except that +# many implementations break this and either use CR or LF alone. +LINE_BREAK = b"(?:\r\n|\n|\r)" +BLANK_LINE_RE = re.compile(b"(?:\r\n\r\n|\r\r|\n\n)", re.MULTILINE) +LINE_BREAK_RE = re.compile(LINE_BREAK, re.MULTILINE) +# Header values can be continued via a space or tab after the linebreak, as +# per RFC2231 +HEADER_CONTINUATION_RE = re.compile(b"%s[ \t]" % LINE_BREAK, re.MULTILINE) +# This must be long enough to contain any line breaks plus any +# additional boundary markers (--) such that they will be found in a +# subsequent search +SEARCH_EXTRA_LENGTH = 8 + + +class MultipartDecoder: + """Decodes a multipart message as bytes into Python events. + + The part data is returned as available to allow the caller to save + the data from memory to disk, if desired. + + .. versionchanged:: 3.1.4 + Handle chunks that split a``\r\n`` sequence. + """ + + def __init__( + self, + boundary: bytes, + max_form_memory_size: int | None = None, + *, + max_parts: int | None = None, + ) -> None: + self.buffer = bytearray() + self.complete = False + self.max_form_memory_size = max_form_memory_size + self.max_parts = max_parts + self.state = State.PREAMBLE + self.boundary = boundary + + # Note in the below \h i.e. horizontal whitespace is used + # as [^\S\n\r] as \h isn't supported in python. + + # The preamble must end with a boundary where the boundary is + # prefixed by a line break, RFC2046. Except that many + # implementations including Werkzeug's tests omit the line + # break prefix. In addition the first boundary could be the + # epilogue boundary (for empty form-data) hence the matching + # group to understand if it is an epilogue boundary. + self.preamble_re = re.compile( + rb"%s?--%s(--[^\S\n\r]*%s?|[^\S\n\r]*%s)" + % (LINE_BREAK, re.escape(boundary), LINE_BREAK, LINE_BREAK), + re.MULTILINE, + ) + # A boundary must include a line break prefix and suffix, and + # may include trailing whitespace. In addition the boundary + # could be the epilogue boundary hence the matching group to + # understand if it is an epilogue boundary. + self.boundary_re = re.compile( + rb"%s--%s(--[^\S\n\r]*%s?|[^\S\n\r]*%s)" + % (LINE_BREAK, re.escape(boundary), LINE_BREAK, LINE_BREAK), + re.MULTILINE, + ) + self._search_position = 0 + self._parts_decoded = 0 + + def receive_data(self, data: bytes | None) -> None: + if data is None: + self.complete = True + elif ( + self.max_form_memory_size is not None + and len(self.buffer) + len(data) > self.max_form_memory_size + ): + # Ensure that data within single event does not exceed limit. + # Also checked across accumulated events in MultiPartParser. + raise RequestEntityTooLarge() + else: + self.buffer.extend(data) + + def next_event(self) -> Event: + event: Event = NEED_DATA + + if self.state == State.PREAMBLE: + match = self.preamble_re.search(self.buffer, self._search_position) + if match is not None: + if match.group(1).startswith(b"--"): + self.state = State.EPILOGUE + else: + self.state = State.PART + data = bytes(self.buffer[: match.start()]) + del self.buffer[: match.end()] + event = Preamble(data=data) + self._search_position = 0 + else: + # Update the search start position to be equal to the + # current buffer length (already searched) minus a + # safe buffer for part of the search target. + self._search_position = max( + 0, len(self.buffer) - len(self.boundary) - SEARCH_EXTRA_LENGTH + ) + + elif self.state == State.PART: + match = BLANK_LINE_RE.search(self.buffer, self._search_position) + if match is not None: + headers = self._parse_headers(self.buffer[: match.start()]) + # The final header ends with a single CRLF, however a + # blank line indicates the start of the + # body. Therefore the end is after the first CRLF. + headers_end = (match.start() + match.end()) // 2 + del self.buffer[:headers_end] + + if "content-disposition" not in headers: + raise ValueError("Missing Content-Disposition header") + + disposition, extra = parse_options_header( + headers["content-disposition"] + ) + name = t.cast(str, extra.get("name")) + filename = extra.get("filename") + if filename is not None: + event = File( + filename=filename, + headers=headers, + name=name, + ) + else: + event = Field( + headers=headers, + name=name, + ) + self.state = State.DATA_START + self._search_position = 0 + self._parts_decoded += 1 + + if self.max_parts is not None and self._parts_decoded > self.max_parts: + raise RequestEntityTooLarge() + else: + # Update the search start position to be equal to the + # current buffer length (already searched) minus a + # safe buffer for part of the search target. + self._search_position = max(0, len(self.buffer) - SEARCH_EXTRA_LENGTH) + + elif self.state == State.DATA_START: + data, del_index, more_data = self._parse_data(self.buffer, start=True) + del self.buffer[:del_index] + event = Data(data=data, more_data=more_data) + if more_data: + self.state = State.DATA + + elif self.state == State.DATA: + data, del_index, more_data = self._parse_data(self.buffer, start=False) + del self.buffer[:del_index] + if data or not more_data: + event = Data(data=data, more_data=more_data) + + elif self.state == State.EPILOGUE and self.complete: + event = Epilogue(data=bytes(self.buffer)) + del self.buffer[:] + self.state = State.COMPLETE + + if self.complete and isinstance(event, NeedData): + raise ValueError(f"Invalid form-data cannot parse beyond {self.state}") + + return event + + def _parse_headers(self, data: bytes | bytearray) -> Headers: + headers: list[tuple[str, str]] = [] + # Merge the continued headers into one line + data = HEADER_CONTINUATION_RE.sub(b" ", data) + # Now there is one header per line + for line in data.splitlines(): + line = line.strip() + + if line != b"": + name, _, value = line.decode().partition(":") + headers.append((name.strip(), value.strip())) + return Headers(headers) + + def _parse_data( + self, data: bytes | bytearray, *, start: bool + ) -> tuple[bytes, int, bool]: + # Body parts must start with CRLF (or CR or LF) + if start: + match = LINE_BREAK_RE.match(data) + data_start = t.cast(t.Match[bytes], match).end() + else: + data_start = 0 + + if self.buffer.find(b"--" + self.boundary) == -1: + # No complete boundary in the buffer, but there may be + # a partial boundary at the end. + data_end = del_index = self._last_partial_boundary_index(data) + more_data = True + else: + match = self.boundary_re.search(data) + if match is not None: + if match.group(1).startswith(b"--"): + self.state = State.EPILOGUE + else: + self.state = State.PART + data_end = match.start() + del_index = match.end() + else: + data_end = del_index = self._last_partial_boundary_index(data) + more_data = match is None + + return bytes(data[data_start:data_end]), del_index, more_data + + def _last_partial_boundary_index(self, data: bytes | bytearray) -> int: + # Find the last index following which a partial boundary + # could be present in the data. This will be the earliest + # position of a LR or a CR, unless that position is more + # than a complete boundary from the end in which case there + # is no partial boundary. + complete_boundary_index = len(data) - len(b"\r\n--" + self.boundary) + try: + last_nl = data.rindex(b"\n") + except ValueError: + last_nl = len(data) + else: + if last_nl < complete_boundary_index: + last_nl = len(data) + try: + last_cr = data.rindex(b"\r") + except ValueError: + last_cr = len(data) + else: + if last_cr < complete_boundary_index: + last_cr = len(data) + return min(last_nl, last_cr) + + +class MultipartEncoder: + def __init__(self, boundary: bytes) -> None: + self.boundary = boundary + self.state = State.PREAMBLE + + def send_event(self, event: Event) -> bytes: + if isinstance(event, Preamble) and self.state == State.PREAMBLE: + self.state = State.PART + return event.data + elif isinstance(event, (Field, File)) and self.state in { + State.PREAMBLE, + State.PART, + State.DATA, + }: + data = b"\r\n--" + self.boundary + b"\r\n" + data += b'Content-Disposition: form-data; name="%s"' % event.name.encode() + if isinstance(event, File): + data += b'; filename="%s"' % event.filename.encode() + data += b"\r\n" + for name, value in t.cast(Field, event).headers: + if name.lower() != "content-disposition": + data += f"{name}: {value}\r\n".encode() + self.state = State.DATA_START + return data + elif isinstance(event, Data) and self.state == State.DATA_START: + self.state = State.DATA + if len(event.data) > 0: + return b"\r\n" + event.data + else: + return event.data + elif isinstance(event, Data) and self.state == State.DATA: + return event.data + elif isinstance(event, Epilogue): + self.state = State.COMPLETE + return b"\r\n--" + self.boundary + b"--\r\n" + event.data + else: + raise ValueError(f"Cannot generate {event} in state: {self.state}") diff --git a/lib/python3.12/site-packages/werkzeug/sansio/request.py b/lib/python3.12/site-packages/werkzeug/sansio/request.py new file mode 100644 index 0000000000000000000000000000000000000000..8d5fbd8f8f2b9cb0c390dc846eab4eaaae5ad586 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/sansio/request.py @@ -0,0 +1,534 @@ +from __future__ import annotations + +import typing as t +from datetime import datetime +from urllib.parse import parse_qsl + +from ..datastructures import Accept +from ..datastructures import Authorization +from ..datastructures import CharsetAccept +from ..datastructures import ETags +from ..datastructures import Headers +from ..datastructures import HeaderSet +from ..datastructures import IfRange +from ..datastructures import ImmutableList +from ..datastructures import ImmutableMultiDict +from ..datastructures import LanguageAccept +from ..datastructures import MIMEAccept +from ..datastructures import MultiDict +from ..datastructures import Range +from ..datastructures import RequestCacheControl +from ..http import parse_accept_header +from ..http import parse_cache_control_header +from ..http import parse_date +from ..http import parse_etags +from ..http import parse_if_range_header +from ..http import parse_list_header +from ..http import parse_options_header +from ..http import parse_range_header +from ..http import parse_set_header +from ..user_agent import UserAgent +from ..utils import cached_property +from ..utils import header_property +from .http import parse_cookie +from .utils import get_content_length +from .utils import get_current_url +from .utils import get_host + + +class Request: + """Represents the non-IO parts of a HTTP request, including the + method, URL info, and headers. + + This class is not meant for general use. It should only be used when + implementing WSGI, ASGI, or another HTTP application spec. Werkzeug + provides a WSGI implementation at :cls:`werkzeug.wrappers.Request`. + + :param method: The method the request was made with, such as + ``GET``. + :param scheme: The URL scheme of the protocol the request used, such + as ``https`` or ``wss``. + :param server: The address of the server. ``(host, port)``, + ``(path, None)`` for unix sockets, or ``None`` if not known. + :param root_path: The prefix that the application is mounted under. + This is prepended to generated URLs, but is not part of route + matching. + :param path: The path part of the URL after ``root_path``. + :param query_string: The part of the URL after the "?". + :param headers: The headers received with the request. + :param remote_addr: The address of the client sending the request. + + .. versionchanged:: 3.0 + The ``charset``, ``url_charset``, and ``encoding_errors`` attributes + were removed. + + .. versionadded:: 2.0 + """ + + #: the class to use for `args` and `form`. The default is an + #: :class:`~werkzeug.datastructures.ImmutableMultiDict` which supports + #: multiple values per key. A :class:`~werkzeug.datastructures.ImmutableDict` + #: is faster but only remembers the last key. It is also + #: possible to use mutable structures, but this is not recommended. + #: + #: .. versionadded:: 0.6 + parameter_storage_class: type[MultiDict[str, t.Any]] = ImmutableMultiDict + + #: The type to be used for dict values from the incoming WSGI + #: environment. (For example for :attr:`cookies`.) By default an + #: :class:`~werkzeug.datastructures.ImmutableMultiDict` is used. + #: + #: .. versionchanged:: 1.0.0 + #: Changed to ``ImmutableMultiDict`` to support multiple values. + #: + #: .. versionadded:: 0.6 + dict_storage_class: type[MultiDict[str, t.Any]] = ImmutableMultiDict + + #: the type to be used for list values from the incoming WSGI environment. + #: By default an :class:`~werkzeug.datastructures.ImmutableList` is used + #: (for example for :attr:`access_list`). + #: + #: .. versionadded:: 0.6 + list_storage_class: type[list[t.Any]] = ImmutableList + + user_agent_class: type[UserAgent] = UserAgent + """The class used and returned by the :attr:`user_agent` property to + parse the header. Defaults to + :class:`~werkzeug.user_agent.UserAgent`, which does no parsing. An + extension can provide a subclass that uses a parser to provide other + data. + + .. versionadded:: 2.0 + """ + + #: Valid host names when handling requests. By default all hosts are + #: trusted, which means that whatever the client says the host is + #: will be accepted. + #: + #: Because ``Host`` and ``X-Forwarded-Host`` headers can be set to + #: any value by a malicious client, it is recommended to either set + #: this property or implement similar validation in the proxy (if + #: the application is being run behind one). + #: + #: .. versionadded:: 0.9 + trusted_hosts: list[str] | None = None + + def __init__( + self, + method: str, + scheme: str, + server: tuple[str, int | None] | None, + root_path: str, + path: str, + query_string: bytes, + headers: Headers, + remote_addr: str | None, + ) -> None: + #: The method the request was made with, such as ``GET``. + self.method = method.upper() + #: The URL scheme of the protocol the request used, such as + #: ``https`` or ``wss``. + self.scheme = scheme + #: The address of the server. ``(host, port)``, ``(path, None)`` + #: for unix sockets, or ``None`` if not known. + self.server = server + #: The prefix that the application is mounted under, without a + #: trailing slash. :attr:`path` comes after this. + self.root_path = root_path.rstrip("/") + #: The path part of the URL after :attr:`root_path`. This is the + #: path used for routing within the application. + self.path = "/" + path.lstrip("/") + #: The part of the URL after the "?". This is the raw value, use + #: :attr:`args` for the parsed values. + self.query_string = query_string + #: The headers received with the request. + self.headers = headers + #: The address of the client sending the request. + self.remote_addr = remote_addr + + def __repr__(self) -> str: + try: + url = self.url + except Exception as e: + url = f"(invalid URL: {e})" + + return f"<{type(self).__name__} {url!r} [{self.method}]>" + + @cached_property + def args(self) -> MultiDict[str, str]: + """The parsed URL parameters (the part in the URL after the question + mark). + + By default an + :class:`~werkzeug.datastructures.ImmutableMultiDict` + is returned from this function. This can be changed by setting + :attr:`parameter_storage_class` to a different type. This might + be necessary if the order of the form data is important. + + .. versionchanged:: 2.3 + Invalid bytes remain percent encoded. + """ + return self.parameter_storage_class( + parse_qsl( + self.query_string.decode(), + keep_blank_values=True, + errors="werkzeug.url_quote", + ) + ) + + @cached_property + def access_route(self) -> list[str]: + """If a forwarded header exists this is a list of all ip addresses + from the client ip to the last proxy server. + """ + if "X-Forwarded-For" in self.headers: + return self.list_storage_class( + parse_list_header(self.headers["X-Forwarded-For"]) + ) + elif self.remote_addr is not None: + return self.list_storage_class([self.remote_addr]) + return self.list_storage_class() + + @cached_property + def full_path(self) -> str: + """Requested path, including the query string.""" + return f"{self.path}?{self.query_string.decode()}" + + @property + def is_secure(self) -> bool: + """``True`` if the request was made with a secure protocol + (HTTPS or WSS). + """ + return self.scheme in {"https", "wss"} + + @cached_property + def url(self) -> str: + """The full request URL with the scheme, host, root path, path, + and query string.""" + return get_current_url( + self.scheme, self.host, self.root_path, self.path, self.query_string + ) + + @cached_property + def base_url(self) -> str: + """Like :attr:`url` but without the query string.""" + return get_current_url(self.scheme, self.host, self.root_path, self.path) + + @cached_property + def root_url(self) -> str: + """The request URL scheme, host, and root path. This is the root + that the application is accessed from. + """ + return get_current_url(self.scheme, self.host, self.root_path) + + @cached_property + def host_url(self) -> str: + """The request URL scheme and host only.""" + return get_current_url(self.scheme, self.host) + + @cached_property + def host(self) -> str: + """The host name the request was made to, including the port if + it's non-standard. Validated with :attr:`trusted_hosts`. + """ + return get_host( + self.scheme, self.headers.get("host"), self.server, self.trusted_hosts + ) + + @cached_property + def cookies(self) -> ImmutableMultiDict[str, str]: + """A :class:`dict` with the contents of all cookies transmitted with + the request.""" + wsgi_combined_cookie = ";".join(self.headers.getlist("Cookie")) + return parse_cookie( # type: ignore + wsgi_combined_cookie, cls=self.dict_storage_class + ) + + # Common Descriptors + + content_type = header_property[str]( + "Content-Type", + doc="""The Content-Type entity-header field indicates the media + type of the entity-body sent to the recipient or, in the case of + the HEAD method, the media type that would have been sent had + the request been a GET.""", + read_only=True, + ) + + @cached_property + def content_length(self) -> int | None: + """The Content-Length entity-header field indicates the size of the + entity-body in bytes or, in the case of the HEAD method, the size of + the entity-body that would have been sent had the request been a + GET. + """ + return get_content_length( + http_content_length=self.headers.get("Content-Length"), + http_transfer_encoding=self.headers.get("Transfer-Encoding"), + ) + + content_encoding = header_property[str]( + "Content-Encoding", + doc="""The Content-Encoding entity-header field is used as a + modifier to the media-type. When present, its value indicates + what additional content codings have been applied to the + entity-body, and thus what decoding mechanisms must be applied + in order to obtain the media-type referenced by the Content-Type + header field. + + .. versionadded:: 0.9""", + read_only=True, + ) + content_md5 = header_property[str]( + "Content-MD5", + doc="""The Content-MD5 entity-header field, as defined in + RFC 1864, is an MD5 digest of the entity-body for the purpose of + providing an end-to-end message integrity check (MIC) of the + entity-body. (Note: a MIC is good for detecting accidental + modification of the entity-body in transit, but is not proof + against malicious attacks.) + + .. versionadded:: 0.9""", + read_only=True, + ) + referrer = header_property[str]( + "Referer", + doc="""The Referer[sic] request-header field allows the client + to specify, for the server's benefit, the address (URI) of the + resource from which the Request-URI was obtained (the + "referrer", although the header field is misspelled).""", + read_only=True, + ) + date = header_property( + "Date", + None, + parse_date, + doc="""The Date general-header field represents the date and + time at which the message was originated, having the same + semantics as orig-date in RFC 822. + + .. versionchanged:: 2.0 + The datetime object is timezone-aware. + """, + read_only=True, + ) + max_forwards = header_property( + "Max-Forwards", + None, + int, + doc="""The Max-Forwards request-header field provides a + mechanism with the TRACE and OPTIONS methods to limit the number + of proxies or gateways that can forward the request to the next + inbound server.""", + read_only=True, + ) + + def _parse_content_type(self) -> None: + if not hasattr(self, "_parsed_content_type"): + self._parsed_content_type = parse_options_header( + self.headers.get("Content-Type", "") + ) + + @property + def mimetype(self) -> str: + """Like :attr:`content_type`, but without parameters (eg, without + charset, type etc.) and always lowercase. For example if the content + type is ``text/HTML; charset=utf-8`` the mimetype would be + ``'text/html'``. + """ + self._parse_content_type() + return self._parsed_content_type[0].lower() + + @property + def mimetype_params(self) -> dict[str, str]: + """The mimetype parameters as dict. For example if the content + type is ``text/html; charset=utf-8`` the params would be + ``{'charset': 'utf-8'}``. + """ + self._parse_content_type() + return self._parsed_content_type[1] + + @cached_property + def pragma(self) -> HeaderSet: + """The Pragma general-header field is used to include + implementation-specific directives that might apply to any recipient + along the request/response chain. All pragma directives specify + optional behavior from the viewpoint of the protocol; however, some + systems MAY require that behavior be consistent with the directives. + """ + return parse_set_header(self.headers.get("Pragma", "")) + + # Accept + + @cached_property + def accept_mimetypes(self) -> MIMEAccept: + """List of mimetypes this client supports as + :class:`~werkzeug.datastructures.MIMEAccept` object. + """ + return parse_accept_header(self.headers.get("Accept"), MIMEAccept) + + @cached_property + def accept_charsets(self) -> CharsetAccept: + """List of charsets this client supports as + :class:`~werkzeug.datastructures.CharsetAccept` object. + """ + return parse_accept_header(self.headers.get("Accept-Charset"), CharsetAccept) + + @cached_property + def accept_encodings(self) -> Accept: + """List of encodings this client accepts. Encodings in a HTTP term + are compression encodings such as gzip. For charsets have a look at + :attr:`accept_charset`. + """ + return parse_accept_header(self.headers.get("Accept-Encoding")) + + @cached_property + def accept_languages(self) -> LanguageAccept: + """List of languages this client accepts as + :class:`~werkzeug.datastructures.LanguageAccept` object. + + .. versionchanged 0.5 + In previous versions this was a regular + :class:`~werkzeug.datastructures.Accept` object. + """ + return parse_accept_header(self.headers.get("Accept-Language"), LanguageAccept) + + # ETag + + @cached_property + def cache_control(self) -> RequestCacheControl: + """A :class:`~werkzeug.datastructures.RequestCacheControl` object + for the incoming cache control headers. + """ + cache_control = self.headers.get("Cache-Control") + return parse_cache_control_header(cache_control, None, RequestCacheControl) + + @cached_property + def if_match(self) -> ETags: + """An object containing all the etags in the `If-Match` header. + + :rtype: :class:`~werkzeug.datastructures.ETags` + """ + return parse_etags(self.headers.get("If-Match")) + + @cached_property + def if_none_match(self) -> ETags: + """An object containing all the etags in the `If-None-Match` header. + + :rtype: :class:`~werkzeug.datastructures.ETags` + """ + return parse_etags(self.headers.get("If-None-Match")) + + @cached_property + def if_modified_since(self) -> datetime | None: + """The parsed `If-Modified-Since` header as a datetime object. + + .. versionchanged:: 2.0 + The datetime object is timezone-aware. + """ + return parse_date(self.headers.get("If-Modified-Since")) + + @cached_property + def if_unmodified_since(self) -> datetime | None: + """The parsed `If-Unmodified-Since` header as a datetime object. + + .. versionchanged:: 2.0 + The datetime object is timezone-aware. + """ + return parse_date(self.headers.get("If-Unmodified-Since")) + + @cached_property + def if_range(self) -> IfRange: + """The parsed ``If-Range`` header. + + .. versionchanged:: 2.0 + ``IfRange.date`` is timezone-aware. + + .. versionadded:: 0.7 + """ + return parse_if_range_header(self.headers.get("If-Range")) + + @cached_property + def range(self) -> Range | None: + """The parsed `Range` header. + + .. versionadded:: 0.7 + + :rtype: :class:`~werkzeug.datastructures.Range` + """ + return parse_range_header(self.headers.get("Range")) + + # User Agent + + @cached_property + def user_agent(self) -> UserAgent: + """The user agent. Use ``user_agent.string`` to get the header + value. Set :attr:`user_agent_class` to a subclass of + :class:`~werkzeug.user_agent.UserAgent` to provide parsing for + the other properties or other extended data. + + .. versionchanged:: 2.1 + The built-in parser was removed. Set ``user_agent_class`` to a ``UserAgent`` + subclass to parse data from the string. + """ + return self.user_agent_class(self.headers.get("User-Agent", "")) + + # Authorization + + @cached_property + def authorization(self) -> Authorization | None: + """The ``Authorization`` header parsed into an :class:`.Authorization` object. + ``None`` if the header is not present. + + .. versionchanged:: 2.3 + :class:`Authorization` is no longer a ``dict``. The ``token`` attribute + was added for auth schemes that use a token instead of parameters. + """ + return Authorization.from_header(self.headers.get("Authorization")) + + # CORS + + origin = header_property[str]( + "Origin", + doc=( + "The host that the request originated from. Set" + " :attr:`~CORSResponseMixin.access_control_allow_origin` on" + " the response to indicate which origins are allowed." + ), + read_only=True, + ) + + access_control_request_headers = header_property( + "Access-Control-Request-Headers", + load_func=parse_set_header, + doc=( + "Sent with a preflight request to indicate which headers" + " will be sent with the cross origin request. Set" + " :attr:`~CORSResponseMixin.access_control_allow_headers`" + " on the response to indicate which headers are allowed." + ), + read_only=True, + ) + + access_control_request_method = header_property[str]( + "Access-Control-Request-Method", + doc=( + "Sent with a preflight request to indicate which method" + " will be used for the cross origin request. Set" + " :attr:`~CORSResponseMixin.access_control_allow_methods`" + " on the response to indicate which methods are allowed." + ), + read_only=True, + ) + + @property + def is_json(self) -> bool: + """Check if the mimetype indicates JSON data, either + :mimetype:`application/json` or :mimetype:`application/*+json`. + """ + mt = self.mimetype + return ( + mt == "application/json" + or mt.startswith("application/") + and mt.endswith("+json") + ) diff --git a/lib/python3.12/site-packages/werkzeug/sansio/response.py b/lib/python3.12/site-packages/werkzeug/sansio/response.py new file mode 100644 index 0000000000000000000000000000000000000000..a52f963de2e0f697a588c36851982028c6bcb04a --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/sansio/response.py @@ -0,0 +1,763 @@ +from __future__ import annotations + +import typing as t +from datetime import datetime +from datetime import timedelta +from datetime import timezone +from http import HTTPStatus + +from ..datastructures import CallbackDict +from ..datastructures import ContentRange +from ..datastructures import ContentSecurityPolicy +from ..datastructures import Headers +from ..datastructures import HeaderSet +from ..datastructures import ResponseCacheControl +from ..datastructures import WWWAuthenticate +from ..http import COEP +from ..http import COOP +from ..http import dump_age +from ..http import dump_cookie +from ..http import dump_header +from ..http import dump_options_header +from ..http import http_date +from ..http import HTTP_STATUS_CODES +from ..http import parse_age +from ..http import parse_cache_control_header +from ..http import parse_content_range_header +from ..http import parse_csp_header +from ..http import parse_date +from ..http import parse_options_header +from ..http import parse_set_header +from ..http import quote_etag +from ..http import unquote_etag +from ..utils import get_content_type +from ..utils import header_property + +if t.TYPE_CHECKING: + from ..datastructures.cache_control import _CacheControl + + +def _set_property(name: str, doc: str | None = None) -> property: + def fget(self: Response) -> HeaderSet: + def on_update(header_set: HeaderSet) -> None: + if not header_set and name in self.headers: + del self.headers[name] + elif header_set: + self.headers[name] = header_set.to_header() + + return parse_set_header(self.headers.get(name), on_update) + + def fset( + self: Response, + value: None | (str | dict[str, str | int] | t.Iterable[str]), + ) -> None: + if not value: + del self.headers[name] + elif isinstance(value, str): + self.headers[name] = value + else: + self.headers[name] = dump_header(value) + + return property(fget, fset, doc=doc) + + +class Response: + """Represents the non-IO parts of an HTTP response, specifically the + status and headers but not the body. + + This class is not meant for general use. It should only be used when + implementing WSGI, ASGI, or another HTTP application spec. Werkzeug + provides a WSGI implementation at :cls:`werkzeug.wrappers.Response`. + + :param status: The status code for the response. Either an int, in + which case the default status message is added, or a string in + the form ``{code} {message}``, like ``404 Not Found``. Defaults + to 200. + :param headers: A :class:`~werkzeug.datastructures.Headers` object, + or a list of ``(key, value)`` tuples that will be converted to a + ``Headers`` object. + :param mimetype: The mime type (content type without charset or + other parameters) of the response. If the value starts with + ``text/`` (or matches some other special cases), the charset + will be added to create the ``content_type``. + :param content_type: The full content type of the response. + Overrides building the value from ``mimetype``. + + .. versionchanged:: 3.0 + The ``charset`` attribute was removed. + + .. versionadded:: 2.0 + """ + + #: the default status if none is provided. + default_status = 200 + + #: the default mimetype if none is provided. + default_mimetype: str | None = "text/plain" + + #: Warn if a cookie header exceeds this size. The default, 4093, should be + #: safely `supported by most browsers `_. A cookie larger than + #: this size will still be sent, but it may be ignored or handled + #: incorrectly by some browsers. Set to 0 to disable this check. + #: + #: .. versionadded:: 0.13 + #: + #: .. _`cookie`: http://browsercookielimits.squawky.net/ + max_cookie_size = 4093 + + # A :class:`Headers` object representing the response headers. + headers: Headers + + def __init__( + self, + status: int | str | HTTPStatus | None = None, + headers: t.Mapping[str, str | t.Iterable[str]] + | t.Iterable[tuple[str, str]] + | None = None, + mimetype: str | None = None, + content_type: str | None = None, + ) -> None: + if isinstance(headers, Headers): + self.headers = headers + elif not headers: + self.headers = Headers() + else: + self.headers = Headers(headers) + + if content_type is None: + if mimetype is None and "content-type" not in self.headers: + mimetype = self.default_mimetype + if mimetype is not None: + mimetype = get_content_type(mimetype, "utf-8") + content_type = mimetype + if content_type is not None: + self.headers["Content-Type"] = content_type + if status is None: + status = self.default_status + self.status = status + + def __repr__(self) -> str: + return f"<{type(self).__name__} [{self.status}]>" + + @property + def status_code(self) -> int: + """The HTTP status code as a number.""" + return self._status_code + + @status_code.setter + def status_code(self, code: int) -> None: + self.status = code + + @property + def status(self) -> str: + """The HTTP status code as a string.""" + return self._status + + @status.setter + def status(self, value: str | int | HTTPStatus) -> None: + self._status, self._status_code = self._clean_status(value) + + def _clean_status(self, value: str | int | HTTPStatus) -> tuple[str, int]: + if isinstance(value, (int, HTTPStatus)): + status_code = int(value) + else: + value = value.strip() + + if not value: + raise ValueError("Empty status argument") + + code_str, sep, _ = value.partition(" ") + + try: + status_code = int(code_str) + except ValueError: + # only message + return f"0 {value}", 0 + + if sep: + # code and message + return value, status_code + + # only code, look up message + try: + status = f"{status_code} {HTTP_STATUS_CODES[status_code].upper()}" + except KeyError: + status = f"{status_code} UNKNOWN" + + return status, status_code + + def set_cookie( + self, + key: str, + value: str = "", + max_age: timedelta | int | None = None, + expires: str | datetime | int | float | None = None, + path: str | None = "/", + domain: str | None = None, + secure: bool = False, + httponly: bool = False, + samesite: str | None = None, + partitioned: bool = False, + ) -> None: + """Sets a cookie. + + A warning is raised if the size of the cookie header exceeds + :attr:`max_cookie_size`, but the header will still be set. + + :param key: the key (name) of the cookie to be set. + :param value: the value of the cookie. + :param max_age: should be a number of seconds, or `None` (default) if + the cookie should last only as long as the client's + browser session. + :param expires: should be a `datetime` object or UNIX timestamp. + :param path: limits the cookie to a given path, per default it will + span the whole domain. + :param domain: if you want to set a cross-domain cookie. For example, + ``domain="example.com"`` will set a cookie that is + readable by the domain ``www.example.com``, + ``foo.example.com`` etc. Otherwise, a cookie will only + be readable by the domain that set it. + :param secure: If ``True``, the cookie will only be available + via HTTPS. + :param httponly: Disallow JavaScript access to the cookie. + :param samesite: Limit the scope of the cookie to only be + attached to requests that are "same-site". + :param partitioned: If ``True``, the cookie will be partitioned. + + .. versionchanged:: 3.1 + The ``partitioned`` parameter was added. + """ + self.headers.add( + "Set-Cookie", + dump_cookie( + key, + value=value, + max_age=max_age, + expires=expires, + path=path, + domain=domain, + secure=secure, + httponly=httponly, + max_size=self.max_cookie_size, + samesite=samesite, + partitioned=partitioned, + ), + ) + + def delete_cookie( + self, + key: str, + path: str | None = "/", + domain: str | None = None, + secure: bool = False, + httponly: bool = False, + samesite: str | None = None, + partitioned: bool = False, + ) -> None: + """Delete a cookie. Fails silently if key doesn't exist. + + :param key: the key (name) of the cookie to be deleted. + :param path: if the cookie that should be deleted was limited to a + path, the path has to be defined here. + :param domain: if the cookie that should be deleted was limited to a + domain, that domain has to be defined here. + :param secure: If ``True``, the cookie will only be available + via HTTPS. + :param httponly: Disallow JavaScript access to the cookie. + :param samesite: Limit the scope of the cookie to only be + attached to requests that are "same-site". + :param partitioned: If ``True``, the cookie will be partitioned. + """ + self.set_cookie( + key, + expires=0, + max_age=0, + path=path, + domain=domain, + secure=secure, + httponly=httponly, + samesite=samesite, + partitioned=partitioned, + ) + + @property + def is_json(self) -> bool: + """Check if the mimetype indicates JSON data, either + :mimetype:`application/json` or :mimetype:`application/*+json`. + """ + mt = self.mimetype + return mt is not None and ( + mt == "application/json" + or mt.startswith("application/") + and mt.endswith("+json") + ) + + # Common Descriptors + + @property + def mimetype(self) -> str | None: + """The mimetype (content type without charset etc.)""" + ct = self.headers.get("content-type") + + if ct: + return ct.split(";")[0].strip() + else: + return None + + @mimetype.setter + def mimetype(self, value: str) -> None: + self.headers["Content-Type"] = get_content_type(value, "utf-8") + + @property + def mimetype_params(self) -> dict[str, str]: + """The mimetype parameters as dict. For example if the + content type is ``text/html; charset=utf-8`` the params would be + ``{'charset': 'utf-8'}``. + + .. versionadded:: 0.5 + """ + + def on_update(d: CallbackDict[str, str]) -> None: + self.headers["Content-Type"] = dump_options_header(self.mimetype, d) + + d = parse_options_header(self.headers.get("content-type", ""))[1] + return CallbackDict(d, on_update) + + location = header_property[str]( + "Location", + doc="""The Location response-header field is used to redirect + the recipient to a location other than the Request-URI for + completion of the request or identification of a new + resource.""", + ) + age = header_property( + "Age", + None, + parse_age, + dump_age, # type: ignore + doc="""The Age response-header field conveys the sender's + estimate of the amount of time since the response (or its + revalidation) was generated at the origin server. + + Age values are non-negative decimal integers, representing time + in seconds.""", + ) + content_type = header_property[str]( + "Content-Type", + doc="""The Content-Type entity-header field indicates the media + type of the entity-body sent to the recipient or, in the case of + the HEAD method, the media type that would have been sent had + the request been a GET.""", + ) + content_length = header_property( + "Content-Length", + None, + int, + str, + doc="""The Content-Length entity-header field indicates the size + of the entity-body, in decimal number of OCTETs, sent to the + recipient or, in the case of the HEAD method, the size of the + entity-body that would have been sent had the request been a + GET.""", + ) + content_location = header_property[str]( + "Content-Location", + doc="""The Content-Location entity-header field MAY be used to + supply the resource location for the entity enclosed in the + message when that entity is accessible from a location separate + from the requested resource's URI.""", + ) + content_encoding = header_property[str]( + "Content-Encoding", + doc="""The Content-Encoding entity-header field is used as a + modifier to the media-type. When present, its value indicates + what additional content codings have been applied to the + entity-body, and thus what decoding mechanisms must be applied + in order to obtain the media-type referenced by the Content-Type + header field.""", + ) + content_md5 = header_property[str]( + "Content-MD5", + doc="""The Content-MD5 entity-header field, as defined in + RFC 1864, is an MD5 digest of the entity-body for the purpose of + providing an end-to-end message integrity check (MIC) of the + entity-body. (Note: a MIC is good for detecting accidental + modification of the entity-body in transit, but is not proof + against malicious attacks.)""", + ) + date = header_property( + "Date", + None, + parse_date, + http_date, + doc="""The Date general-header field represents the date and + time at which the message was originated, having the same + semantics as orig-date in RFC 822. + + .. versionchanged:: 2.0 + The datetime object is timezone-aware. + """, + ) + expires = header_property( + "Expires", + None, + parse_date, + http_date, + doc="""The Expires entity-header field gives the date/time after + which the response is considered stale. A stale cache entry may + not normally be returned by a cache. + + .. versionchanged:: 2.0 + The datetime object is timezone-aware. + """, + ) + last_modified = header_property( + "Last-Modified", + None, + parse_date, + http_date, + doc="""The Last-Modified entity-header field indicates the date + and time at which the origin server believes the variant was + last modified. + + .. versionchanged:: 2.0 + The datetime object is timezone-aware. + """, + ) + + @property + def retry_after(self) -> datetime | None: + """The Retry-After response-header field can be used with a + 503 (Service Unavailable) response to indicate how long the + service is expected to be unavailable to the requesting client. + + Time in seconds until expiration or date. + + .. versionchanged:: 2.0 + The datetime object is timezone-aware. + """ + value = self.headers.get("retry-after") + if value is None: + return None + + try: + seconds = int(value) + except ValueError: + return parse_date(value) + + return datetime.now(timezone.utc) + timedelta(seconds=seconds) + + @retry_after.setter + def retry_after(self, value: datetime | int | str | None) -> None: + if value is None: + if "retry-after" in self.headers: + del self.headers["retry-after"] + return + elif isinstance(value, datetime): + value = http_date(value) + else: + value = str(value) + self.headers["Retry-After"] = value + + vary = _set_property( + "Vary", + doc="""The Vary field value indicates the set of request-header + fields that fully determines, while the response is fresh, + whether a cache is permitted to use the response to reply to a + subsequent request without revalidation.""", + ) + content_language = _set_property( + "Content-Language", + doc="""The Content-Language entity-header field describes the + natural language(s) of the intended audience for the enclosed + entity. Note that this might not be equivalent to all the + languages used within the entity-body.""", + ) + allow = _set_property( + "Allow", + doc="""The Allow entity-header field lists the set of methods + supported by the resource identified by the Request-URI. The + purpose of this field is strictly to inform the recipient of + valid methods associated with the resource. An Allow header + field MUST be present in a 405 (Method Not Allowed) + response.""", + ) + + # ETag + + @property + def cache_control(self) -> ResponseCacheControl: + """The Cache-Control general-header field is used to specify + directives that MUST be obeyed by all caching mechanisms along the + request/response chain. + """ + + def on_update(cache_control: _CacheControl) -> None: + if not cache_control and "cache-control" in self.headers: + del self.headers["cache-control"] + elif cache_control: + self.headers["Cache-Control"] = cache_control.to_header() + + return parse_cache_control_header( + self.headers.get("cache-control"), on_update, ResponseCacheControl + ) + + def set_etag(self, etag: str, weak: bool = False) -> None: + """Set the etag, and override the old one if there was one.""" + self.headers["ETag"] = quote_etag(etag, weak) + + def get_etag(self) -> tuple[str, bool] | tuple[None, None]: + """Return a tuple in the form ``(etag, is_weak)``. If there is no + ETag the return value is ``(None, None)``. + """ + return unquote_etag(self.headers.get("ETag")) + + accept_ranges = header_property[str]( + "Accept-Ranges", + doc="""The `Accept-Ranges` header. Even though the name would + indicate that multiple values are supported, it must be one + string token only. + + The values ``'bytes'`` and ``'none'`` are common. + + .. versionadded:: 0.7""", + ) + + @property + def content_range(self) -> ContentRange: + """The ``Content-Range`` header as a + :class:`~werkzeug.datastructures.ContentRange` object. Available + even if the header is not set. + + .. versionadded:: 0.7 + """ + + def on_update(rng: ContentRange) -> None: + if not rng: + del self.headers["content-range"] + else: + self.headers["Content-Range"] = rng.to_header() + + rv = parse_content_range_header(self.headers.get("content-range"), on_update) + # always provide a content range object to make the descriptor + # more user friendly. It provides an unset() method that can be + # used to remove the header quickly. + if rv is None: + rv = ContentRange(None, None, None, on_update=on_update) + return rv + + @content_range.setter + def content_range(self, value: ContentRange | str | None) -> None: + if not value: + del self.headers["content-range"] + elif isinstance(value, str): + self.headers["Content-Range"] = value + else: + self.headers["Content-Range"] = value.to_header() + + # Authorization + + @property + def www_authenticate(self) -> WWWAuthenticate: + """The ``WWW-Authenticate`` header parsed into a :class:`.WWWAuthenticate` + object. Modifying the object will modify the header value. + + This header is not set by default. To set this header, assign an instance of + :class:`.WWWAuthenticate` to this attribute. + + .. code-block:: python + + response.www_authenticate = WWWAuthenticate( + "basic", {"realm": "Authentication Required"} + ) + + Multiple values for this header can be sent to give the client multiple options. + Assign a list to set multiple headers. However, modifying the items in the list + will not automatically update the header values, and accessing this attribute + will only ever return the first value. + + To unset this header, assign ``None`` or use ``del``. + + .. versionchanged:: 2.3 + This attribute can be assigned to set the header. A list can be assigned + to set multiple header values. Use ``del`` to unset the header. + + .. versionchanged:: 2.3 + :class:`WWWAuthenticate` is no longer a ``dict``. The ``token`` attribute + was added for auth challenges that use a token instead of parameters. + """ + value = WWWAuthenticate.from_header(self.headers.get("WWW-Authenticate")) + + if value is None: + value = WWWAuthenticate("basic") + + def on_update(value: WWWAuthenticate) -> None: + self.www_authenticate = value + + value._on_update = on_update + return value + + @www_authenticate.setter + def www_authenticate( + self, value: WWWAuthenticate | list[WWWAuthenticate] | None + ) -> None: + if not value: # None or empty list + del self.www_authenticate + elif isinstance(value, list): + # Clear any existing header by setting the first item. + self.headers.set("WWW-Authenticate", value[0].to_header()) + + for item in value[1:]: + # Add additional header lines for additional items. + self.headers.add("WWW-Authenticate", item.to_header()) + else: + self.headers.set("WWW-Authenticate", value.to_header()) + + def on_update(value: WWWAuthenticate) -> None: + self.www_authenticate = value + + # When setting a single value, allow updating it directly. + value._on_update = on_update + + @www_authenticate.deleter + def www_authenticate(self) -> None: + if "WWW-Authenticate" in self.headers: + del self.headers["WWW-Authenticate"] + + # CSP + + @property + def content_security_policy(self) -> ContentSecurityPolicy: + """The ``Content-Security-Policy`` header as a + :class:`~werkzeug.datastructures.ContentSecurityPolicy` object. Available + even if the header is not set. + + The Content-Security-Policy header adds an additional layer of + security to help detect and mitigate certain types of attacks. + """ + + def on_update(csp: ContentSecurityPolicy) -> None: + if not csp: + del self.headers["content-security-policy"] + else: + self.headers["Content-Security-Policy"] = csp.to_header() + + rv = parse_csp_header(self.headers.get("content-security-policy"), on_update) + if rv is None: + rv = ContentSecurityPolicy(None, on_update=on_update) + return rv + + @content_security_policy.setter + def content_security_policy( + self, value: ContentSecurityPolicy | str | None + ) -> None: + if not value: + del self.headers["content-security-policy"] + elif isinstance(value, str): + self.headers["Content-Security-Policy"] = value + else: + self.headers["Content-Security-Policy"] = value.to_header() + + @property + def content_security_policy_report_only(self) -> ContentSecurityPolicy: + """The ``Content-Security-policy-report-only`` header as a + :class:`~werkzeug.datastructures.ContentSecurityPolicy` object. Available + even if the header is not set. + + The Content-Security-Policy-Report-Only header adds a csp policy + that is not enforced but is reported thereby helping detect + certain types of attacks. + """ + + def on_update(csp: ContentSecurityPolicy) -> None: + if not csp: + del self.headers["content-security-policy-report-only"] + else: + self.headers["Content-Security-policy-report-only"] = csp.to_header() + + rv = parse_csp_header( + self.headers.get("content-security-policy-report-only"), on_update + ) + if rv is None: + rv = ContentSecurityPolicy(None, on_update=on_update) + return rv + + @content_security_policy_report_only.setter + def content_security_policy_report_only( + self, value: ContentSecurityPolicy | str | None + ) -> None: + if not value: + del self.headers["content-security-policy-report-only"] + elif isinstance(value, str): + self.headers["Content-Security-policy-report-only"] = value + else: + self.headers["Content-Security-policy-report-only"] = value.to_header() + + # CORS + + @property + def access_control_allow_credentials(self) -> bool: + """Whether credentials can be shared by the browser to + JavaScript code. As part of the preflight request it indicates + whether credentials can be used on the cross origin request. + """ + return "Access-Control-Allow-Credentials" in self.headers + + @access_control_allow_credentials.setter + def access_control_allow_credentials(self, value: bool | None) -> None: + if value is True: + self.headers["Access-Control-Allow-Credentials"] = "true" + else: + self.headers.pop("Access-Control-Allow-Credentials", None) + + access_control_allow_headers = header_property( + "Access-Control-Allow-Headers", + load_func=parse_set_header, + dump_func=dump_header, + doc="Which headers can be sent with the cross origin request.", + ) + + access_control_allow_methods = header_property( + "Access-Control-Allow-Methods", + load_func=parse_set_header, + dump_func=dump_header, + doc="Which methods can be used for the cross origin request.", + ) + + access_control_allow_origin = header_property[str]( + "Access-Control-Allow-Origin", + doc="The origin or '*' for any origin that may make cross origin requests.", + ) + + access_control_expose_headers = header_property( + "Access-Control-Expose-Headers", + load_func=parse_set_header, + dump_func=dump_header, + doc="Which headers can be shared by the browser to JavaScript code.", + ) + + access_control_max_age = header_property( + "Access-Control-Max-Age", + load_func=int, + dump_func=str, + doc="The maximum age in seconds the access control settings can be cached for.", + ) + + cross_origin_opener_policy = header_property[COOP]( + "Cross-Origin-Opener-Policy", + load_func=lambda value: COOP(value), + dump_func=lambda value: value.value, + default=COOP.UNSAFE_NONE, + doc="""Allows control over sharing of browsing context group with cross-origin + documents. Values must be a member of the :class:`werkzeug.http.COOP` enum.""", + ) + + cross_origin_embedder_policy = header_property[COEP]( + "Cross-Origin-Embedder-Policy", + load_func=lambda value: COEP(value), + dump_func=lambda value: value.value, + default=COEP.UNSAFE_NONE, + doc="""Prevents a document from loading any cross-origin resources that do not + explicitly grant the document permission. Values must be a member of the + :class:`werkzeug.http.COEP` enum.""", + ) diff --git a/lib/python3.12/site-packages/werkzeug/sansio/utils.py b/lib/python3.12/site-packages/werkzeug/sansio/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ff7ceda340ef2f9ec6bb04a9e14da633d579da9e --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/sansio/utils.py @@ -0,0 +1,167 @@ +from __future__ import annotations + +import typing as t +from urllib.parse import quote + +from .._internal import _plain_int +from ..exceptions import SecurityError +from ..urls import uri_to_iri + + +def host_is_trusted(hostname: str | None, trusted_list: t.Iterable[str]) -> bool: + """Check if a host matches a list of trusted names. + + :param hostname: The name to check. + :param trusted_list: A list of valid names to match. If a name + starts with a dot it will match all subdomains. + + .. versionadded:: 0.9 + """ + if not hostname: + return False + + try: + hostname = hostname.partition(":")[0].encode("idna").decode("ascii") + except UnicodeEncodeError: + return False + + if isinstance(trusted_list, str): + trusted_list = [trusted_list] + + for ref in trusted_list: + if ref.startswith("."): + ref = ref[1:] + suffix_match = True + else: + suffix_match = False + + try: + ref = ref.partition(":")[0].encode("idna").decode("ascii") + except UnicodeEncodeError: + return False + + if ref == hostname or (suffix_match and hostname.endswith(f".{ref}")): + return True + + return False + + +def get_host( + scheme: str, + host_header: str | None, + server: tuple[str, int | None] | None = None, + trusted_hosts: t.Iterable[str] | None = None, +) -> str: + """Return the host for the given parameters. + + This first checks the ``host_header``. If it's not present, then + ``server`` is used. The host will only contain the port if it is + different than the standard port for the protocol. + + Optionally, verify that the host is trusted using + :func:`host_is_trusted` and raise a + :exc:`~werkzeug.exceptions.SecurityError` if it is not. + + :param scheme: The protocol the request used, like ``"https"``. + :param host_header: The ``Host`` header value. + :param server: Address of the server. ``(host, port)``, or + ``(path, None)`` for unix sockets. + :param trusted_hosts: A list of trusted host names. + + :return: Host, with port if necessary. + :raise ~werkzeug.exceptions.SecurityError: If the host is not + trusted. + + .. versionchanged:: 3.1.3 + If ``SERVER_NAME`` is IPv6, it is wrapped in ``[]``. + """ + host = "" + + if host_header is not None: + host = host_header + elif server is not None: + host = server[0] + + # If SERVER_NAME is IPv6, wrap it in [] to match Host header. + # Check for : because domain or IPv4 can't have that. + if ":" in host and host[0] != "[": + host = f"[{host}]" + + if server[1] is not None: + host = f"{host}:{server[1]}" + + if scheme in {"http", "ws"} and host.endswith(":80"): + host = host[:-3] + elif scheme in {"https", "wss"} and host.endswith(":443"): + host = host[:-4] + + if trusted_hosts is not None: + if not host_is_trusted(host, trusted_hosts): + raise SecurityError(f"Host {host!r} is not trusted.") + + return host + + +def get_current_url( + scheme: str, + host: str, + root_path: str | None = None, + path: str | None = None, + query_string: bytes | None = None, +) -> str: + """Recreate the URL for a request. If an optional part isn't + provided, it and subsequent parts are not included in the URL. + + The URL is an IRI, not a URI, so it may contain Unicode characters. + Use :func:`~werkzeug.urls.iri_to_uri` to convert it to ASCII. + + :param scheme: The protocol the request used, like ``"https"``. + :param host: The host the request was made to. See :func:`get_host`. + :param root_path: Prefix that the application is mounted under. This + is prepended to ``path``. + :param path: The path part of the URL after ``root_path``. + :param query_string: The portion of the URL after the "?". + """ + url = [scheme, "://", host] + + if root_path is None: + url.append("/") + return uri_to_iri("".join(url)) + + # safe = https://url.spec.whatwg.org/#url-path-segment-string + # as well as percent for things that are already quoted + url.append(quote(root_path.rstrip("/"), safe="!$&'()*+,/:;=@%")) + url.append("/") + + if path is None: + return uri_to_iri("".join(url)) + + url.append(quote(path.lstrip("/"), safe="!$&'()*+,/:;=@%")) + + if query_string: + url.append("?") + url.append(quote(query_string, safe="!$&'()*+,/:;=?@%")) + + return uri_to_iri("".join(url)) + + +def get_content_length( + http_content_length: str | None = None, + http_transfer_encoding: str | None = None, +) -> int | None: + """Return the ``Content-Length`` header value as an int. If the header is not given + or the ``Transfer-Encoding`` header is ``chunked``, ``None`` is returned to indicate + a streaming request. If the value is not an integer, or negative, 0 is returned. + + :param http_content_length: The Content-Length HTTP header. + :param http_transfer_encoding: The Transfer-Encoding HTTP header. + + .. versionadded:: 2.2 + """ + if http_transfer_encoding == "chunked" or http_content_length is None: + return None + + try: + return max(0, _plain_int(http_content_length)) + except ValueError: + return 0 diff --git a/lib/python3.12/site-packages/werkzeug/security.py b/lib/python3.12/site-packages/werkzeug/security.py new file mode 100644 index 0000000000000000000000000000000000000000..85849796f998bee939649ff800e5e72d16b0cd69 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/security.py @@ -0,0 +1,188 @@ +from __future__ import annotations + +import hashlib +import hmac +import os +import posixpath +import secrets + +SALT_CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789" +DEFAULT_PBKDF2_ITERATIONS = 1_000_000 + +_os_alt_seps: list[str] = list( + sep for sep in [os.sep, os.path.altsep] if sep is not None and sep != "/" +) +# https://chrisdenton.github.io/omnipath/Special%20Dos%20Device%20Names.html +_windows_device_files = { + "AUX", + "CON", + "CONIN$", + "CONOUT$", + *(f"COM{c}" for c in "123456789¹²³"), + *(f"LPT{c}" for c in "123456789¹²³"), + "NUL", + "PRN", +} + + +def gen_salt(length: int) -> str: + """Generate a random string of SALT_CHARS with specified ``length``.""" + if length <= 0: + raise ValueError("Salt length must be at least 1.") + + return "".join(secrets.choice(SALT_CHARS) for _ in range(length)) + + +def _hash_internal(method: str, salt: str, password: str) -> tuple[str, str]: + method, *args = method.split(":") + salt_bytes = salt.encode() + password_bytes = password.encode() + + if method == "scrypt": + if not args: + n = 2**15 + r = 8 + p = 1 + else: + try: + n, r, p = map(int, args) + except ValueError: + raise ValueError("'scrypt' takes 3 arguments.") from None + + maxmem = 132 * n * r * p # ideally 128, but some extra seems needed + return ( + hashlib.scrypt( + password_bytes, salt=salt_bytes, n=n, r=r, p=p, maxmem=maxmem + ).hex(), + f"scrypt:{n}:{r}:{p}", + ) + elif method == "pbkdf2": + len_args = len(args) + + if len_args == 0: + hash_name = "sha256" + iterations = DEFAULT_PBKDF2_ITERATIONS + elif len_args == 1: + hash_name = args[0] + iterations = DEFAULT_PBKDF2_ITERATIONS + elif len_args == 2: + hash_name = args[0] + iterations = int(args[1]) + else: + raise ValueError("'pbkdf2' takes 2 arguments.") + + return ( + hashlib.pbkdf2_hmac( + hash_name, password_bytes, salt_bytes, iterations + ).hex(), + f"pbkdf2:{hash_name}:{iterations}", + ) + else: + raise ValueError(f"Invalid hash method '{method}'.") + + +def generate_password_hash( + password: str, method: str = "scrypt", salt_length: int = 16 +) -> str: + """Securely hash a password for storage. A password can be compared to a stored hash + using :func:`check_password_hash`. + + The following methods are supported: + + - ``scrypt``, the default. The parameters are ``n``, ``r``, and ``p``, the default + is ``scrypt:32768:8:1``. See :func:`hashlib.scrypt`. + - ``pbkdf2``, less secure. The parameters are ``hash_method`` and ``iterations``, + the default is ``pbkdf2:sha256:600000``. See :func:`hashlib.pbkdf2_hmac`. + + Default parameters may be updated to reflect current guidelines, and methods may be + deprecated and removed if they are no longer considered secure. To migrate old + hashes, you may generate a new hash when checking an old hash, or you may contact + users with a link to reset their password. + + :param password: The plaintext password. + :param method: The key derivation function and parameters. + :param salt_length: The number of characters to generate for the salt. + + .. versionchanged:: 3.1 + The default iterations for pbkdf2 was increased to 1,000,000. + + .. versionchanged:: 2.3 + Scrypt support was added. + + .. versionchanged:: 2.3 + The default iterations for pbkdf2 was increased to 600,000. + + .. versionchanged:: 2.3 + All plain hashes are deprecated and will not be supported in Werkzeug 3.0. + """ + salt = gen_salt(salt_length) + h, actual_method = _hash_internal(method, salt, password) + return f"{actual_method}${salt}${h}" + + +def check_password_hash(pwhash: str, password: str) -> bool: + """Securely check that the given stored password hash, previously generated using + :func:`generate_password_hash`, matches the given password. + + Methods may be deprecated and removed if they are no longer considered secure. To + migrate old hashes, you may generate a new hash when checking an old hash, or you + may contact users with a link to reset their password. + + :param pwhash: The hashed password. + :param password: The plaintext password. + + .. versionchanged:: 2.3 + All plain hashes are deprecated and will not be supported in Werkzeug 3.0. + """ + try: + method, salt, hashval = pwhash.split("$", 2) + except ValueError: + return False + + return hmac.compare_digest(_hash_internal(method, salt, password)[0], hashval) + + +def safe_join(directory: str, *pathnames: str) -> str | None: + """Safely join zero or more untrusted path components to a base + directory to avoid escaping the base directory. + + :param directory: The trusted base directory. + :param pathnames: The untrusted path components relative to the + base directory. + :return: A safe path, otherwise ``None``. + + .. versionchanged:: 3.1.5 + More special device names, regardless of extension or trailing spaces, + are not allowed on Windows. + + .. versionchanged:: 3.1.4 + Special device names are not allowed on Windows. + """ + if not directory: + # Ensure we end up with ./path if directory="" is given, + # otherwise the first untrusted part could become trusted. + directory = "." + + parts = [directory] + + for filename in pathnames: + if filename != "": + filename = posixpath.normpath(filename) + + if ( + any(sep in filename for sep in _os_alt_seps) + or ( + os.name == "nt" + and filename.partition(".")[0].strip().upper() in _windows_device_files + ) + or os.path.isabs(filename) + # ntpath.isabs doesn't catch this on Python < 3.11 + or filename.startswith("/") + or filename == ".." + or filename.startswith("../") + ): + return None + + parts.append(filename) + + return posixpath.join(*parts) diff --git a/lib/python3.12/site-packages/werkzeug/serving.py b/lib/python3.12/site-packages/werkzeug/serving.py new file mode 100644 index 0000000000000000000000000000000000000000..8605cfd6dfb0cabbc3700059c8429ce74cbf977f --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/serving.py @@ -0,0 +1,1125 @@ +"""A WSGI and HTTP server for use **during development only**. This +server is convenient to use, but is not designed to be particularly +stable, secure, or efficient. Use a dedicate WSGI server and HTTP +server when deploying to production. + +It provides features like interactive debugging and code reloading. Use +``run_simple`` to start the server. Put this in a ``run.py`` script: + +.. code-block:: python + + from myapp import create_app + from werkzeug import run_simple +""" + +from __future__ import annotations + +import errno +import io +import os +import selectors +import socket +import socketserver +import sys +import typing as t +from datetime import datetime as dt +from datetime import timedelta +from datetime import timezone +from http.server import BaseHTTPRequestHandler +from http.server import HTTPServer +from urllib.parse import unquote +from urllib.parse import urlsplit + +from ._internal import _log +from ._internal import _wsgi_encoding_dance +from .exceptions import InternalServerError +from .urls import uri_to_iri + +try: + import ssl + + connection_dropped_errors: tuple[type[Exception], ...] = ( + ConnectionError, + socket.timeout, + ssl.SSLEOFError, + ) +except ImportError: + + class _SslDummy: + def __getattr__(self, name: str) -> t.Any: + raise RuntimeError( # noqa: B904 + "SSL is unavailable because this Python runtime was not" + " compiled with SSL/TLS support." + ) + + ssl = _SslDummy() # type: ignore + connection_dropped_errors = (ConnectionError, socket.timeout) + +_log_add_style = True + +if os.name == "nt": + try: + __import__("colorama") + except ImportError: + _log_add_style = False + +can_fork = hasattr(os, "fork") + +if can_fork: + ForkingMixIn = socketserver.ForkingMixIn +else: + + class ForkingMixIn: # type: ignore + pass + + +try: + af_unix = socket.AF_UNIX +except AttributeError: + af_unix = None # type: ignore + +LISTEN_QUEUE = 128 + +_TSSLContextArg = t.Optional[ + t.Union["ssl.SSLContext", tuple[str, t.Optional[str]], t.Literal["adhoc"]] +] + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + from cryptography.hazmat.primitives.asymmetric.rsa import ( + RSAPrivateKeyWithSerialization, + ) + from cryptography.x509 import Certificate + + +class DechunkedInput(io.RawIOBase): + """An input stream that handles Transfer-Encoding 'chunked'""" + + def __init__(self, rfile: t.IO[bytes]) -> None: + self._rfile = rfile + self._done = False + self._len = 0 + + def readable(self) -> bool: + return True + + def read_chunk_len(self) -> int: + try: + line = self._rfile.readline().decode("latin1") + _len = int(line.strip(), 16) + except ValueError as e: + raise OSError("Invalid chunk header") from e + if _len < 0: + raise OSError("Negative chunk length not allowed") + return _len + + def readinto(self, buf: bytearray) -> int: # type: ignore + read = 0 + while not self._done and read < len(buf): + if self._len == 0: + # This is the first chunk or we fully consumed the previous + # one. Read the next length of the next chunk + self._len = self.read_chunk_len() + + if self._len == 0: + # Found the final chunk of size 0. The stream is now exhausted, + # but there is still a final newline that should be consumed + self._done = True + + if self._len > 0: + # There is data (left) in this chunk, so append it to the + # buffer. If this operation fully consumes the chunk, this will + # reset self._len to 0. + n = min(len(buf), self._len) + + # If (read + chunk size) becomes more than len(buf), buf will + # grow beyond the original size and read more data than + # required. So only read as much data as can fit in buf. + if read + n > len(buf): + buf[read:] = self._rfile.read(len(buf) - read) + self._len -= len(buf) - read + read = len(buf) + else: + buf[read : read + n] = self._rfile.read(n) + self._len -= n + read += n + + if self._len == 0: + # Skip the terminating newline of a chunk that has been fully + # consumed. This also applies to the 0-sized final chunk + terminator = self._rfile.readline() + if terminator not in (b"\n", b"\r\n", b"\r"): + raise OSError("Missing chunk terminating newline") + + return read + + +class WSGIRequestHandler(BaseHTTPRequestHandler): + """A request handler that implements WSGI dispatching.""" + + server: BaseWSGIServer + + @property + def server_version(self) -> str: # type: ignore + return self.server._server_version + + def make_environ(self) -> WSGIEnvironment: + request_url = urlsplit(self.path) + url_scheme = "http" if self.server.ssl_context is None else "https" + + if not self.client_address: + self.client_address = ("", 0) + elif isinstance(self.client_address, str): + self.client_address = (self.client_address, 0) + + # If there was no scheme but the path started with two slashes, + # the first segment may have been incorrectly parsed as the + # netloc, prepend it to the path again. + if not request_url.scheme and request_url.netloc: + path_info = f"/{request_url.netloc}{request_url.path}" + else: + path_info = request_url.path + + path_info = unquote(path_info) + + environ: WSGIEnvironment = { + "wsgi.version": (1, 0), + "wsgi.url_scheme": url_scheme, + "wsgi.input": self.rfile, + "wsgi.errors": sys.stderr, + "wsgi.multithread": self.server.multithread, + "wsgi.multiprocess": self.server.multiprocess, + "wsgi.run_once": False, + "werkzeug.socket": self.connection, + "SERVER_SOFTWARE": self.server_version, + "REQUEST_METHOD": self.command, + "SCRIPT_NAME": "", + "PATH_INFO": _wsgi_encoding_dance(path_info), + "QUERY_STRING": _wsgi_encoding_dance(request_url.query), + # Non-standard, added by mod_wsgi, uWSGI + "REQUEST_URI": _wsgi_encoding_dance(self.path), + # Non-standard, added by gunicorn + "RAW_URI": _wsgi_encoding_dance(self.path), + "REMOTE_ADDR": self.address_string(), + "REMOTE_PORT": self.port_integer(), + "SERVER_NAME": self.server.server_address[0], + "SERVER_PORT": str(self.server.server_address[1]), + "SERVER_PROTOCOL": self.request_version, + } + + for key, value in self.headers.items(): + if "_" in key: + continue + + key = key.upper().replace("-", "_") + value = value.replace("\r\n", "") + if key not in ("CONTENT_TYPE", "CONTENT_LENGTH"): + key = f"HTTP_{key}" + if key in environ: + value = f"{environ[key]},{value}" + environ[key] = value + + if environ.get("HTTP_TRANSFER_ENCODING", "").strip().lower() == "chunked": + environ["wsgi.input_terminated"] = True + environ["wsgi.input"] = DechunkedInput(environ["wsgi.input"]) + + # Per RFC 2616, if the URL is absolute, use that as the host. + # We're using "has a scheme" to indicate an absolute URL. + if request_url.scheme and request_url.netloc: + environ["HTTP_HOST"] = request_url.netloc + + try: + # binary_form=False gives nicer information, but wouldn't be compatible with + # what Nginx or Apache could return. + peer_cert = self.connection.getpeercert(binary_form=True) + if peer_cert is not None: + # Nginx and Apache use PEM format. + environ["SSL_CLIENT_CERT"] = ssl.DER_cert_to_PEM_cert(peer_cert) + except ValueError: + # SSL handshake hasn't finished. + self.server.log("error", "Cannot fetch SSL peer certificate info") + except AttributeError: + # Not using TLS, the socket will not have getpeercert(). + pass + + return environ + + def run_wsgi(self) -> None: + if self.headers.get("Expect", "").lower().strip() == "100-continue": + self.wfile.write(b"HTTP/1.1 100 Continue\r\n\r\n") + + self.environ = environ = self.make_environ() + status_set: str | None = None + headers_set: list[tuple[str, str]] | None = None + status_sent: str | None = None + headers_sent: list[tuple[str, str]] | None = None + chunk_response: bool = False + + def write(data: bytes) -> None: + nonlocal status_sent, headers_sent, chunk_response + assert status_set is not None, "write() before start_response" + assert headers_set is not None, "write() before start_response" + if status_sent is None: + status_sent = status_set + headers_sent = headers_set + try: + code_str, msg = status_sent.split(None, 1) + except ValueError: + code_str, msg = status_sent, "" + code = int(code_str) + self.send_response(code, msg) + header_keys = set() + for key, value in headers_sent: + self.send_header(key, value) + header_keys.add(key.lower()) + + # Use chunked transfer encoding if there is no content + # length. Do not use for 1xx and 204 responses. 304 + # responses and HEAD requests are also excluded, which + # is the more conservative behavior and matches other + # parts of the code. + # https://httpwg.org/specs/rfc7230.html#rfc.section.3.3.1 + if ( + not ( + "content-length" in header_keys + or environ["REQUEST_METHOD"] == "HEAD" + or (100 <= code < 200) + or code in {204, 304} + ) + and self.protocol_version >= "HTTP/1.1" + ): + chunk_response = True + self.send_header("Transfer-Encoding", "chunked") + + # Always close the connection. This disables HTTP/1.1 + # keep-alive connections. They aren't handled well by + # Python's http.server because it doesn't know how to + # drain the stream before the next request line. + self.send_header("Connection", "close") + self.end_headers() + + assert isinstance(data, bytes), "applications must write bytes" + + if data: + if chunk_response: + self.wfile.write(hex(len(data))[2:].encode()) + self.wfile.write(b"\r\n") + + self.wfile.write(data) + + if chunk_response: + self.wfile.write(b"\r\n") + + self.wfile.flush() + + def start_response(status, headers, exc_info=None): # type: ignore + nonlocal status_set, headers_set + if exc_info: + try: + if headers_sent: + raise exc_info[1].with_traceback(exc_info[2]) + finally: + exc_info = None + elif headers_set: + raise AssertionError("Headers already set") + status_set = status + headers_set = headers + return write + + def execute(app: WSGIApplication) -> None: + application_iter = app(environ, start_response) + try: + for data in application_iter: + write(data) + if not headers_sent: + write(b"") + if chunk_response: + self.wfile.write(b"0\r\n\r\n") + finally: + # Check for any remaining data in the read socket, and discard it. This + # will read past request.max_content_length, but lets the client see a + # 413 response instead of a connection reset failure. If we supported + # keep-alive connections, this naive approach would break by reading the + # next request line. Since we know that write (above) closes every + # connection we can read everything. + selector = selectors.DefaultSelector() + selector.register(self.connection, selectors.EVENT_READ) + total_size = 0 + total_reads = 0 + + # A timeout of 0 tends to fail because a client needs a small amount of + # time to continue sending its data. + while selector.select(timeout=0.01): + # Only read 10MB into memory at a time. + data = self.rfile.read(10_000_000) + total_size += len(data) + total_reads += 1 + + # Stop reading on no data, >=10GB, or 1000 reads. If a client sends + # more than that, they'll get a connection reset failure. + if not data or total_size >= 10_000_000_000 or total_reads > 1000: + break + + selector.close() + + if hasattr(application_iter, "close"): + application_iter.close() + + try: + execute(self.server.app) + except connection_dropped_errors as e: + self.connection_dropped(e, environ) + except Exception as e: + if self.server.passthrough_errors: + raise + + if status_sent is not None and chunk_response: + self.close_connection = True + + try: + # if we haven't yet sent the headers but they are set + # we roll back to be able to set them again. + if status_sent is None: + status_set = None + headers_set = None + execute(InternalServerError()) + except Exception: + pass + + from .debug.tbtools import DebugTraceback + + msg = DebugTraceback(e).render_traceback_text() + self.server.log("error", f"Error on request:\n{msg}") + + def handle(self) -> None: + """Handles a request ignoring dropped connections.""" + try: + super().handle() + except (ConnectionError, socket.timeout) as e: + self.connection_dropped(e) + except Exception as e: + if self.server.ssl_context is not None and is_ssl_error(e): + self.log_error("SSL error occurred: %s", e) + else: + raise + + def connection_dropped( + self, error: BaseException, environ: WSGIEnvironment | None = None + ) -> None: + """Called if the connection was closed by the client. By default + nothing happens. + """ + + def __getattr__(self, name: str) -> t.Any: + # All HTTP methods are handled by run_wsgi. + if name.startswith("do_"): + return self.run_wsgi + + # All other attributes are forwarded to the base class. + return getattr(super(), name) + + def address_string(self) -> str: + if getattr(self, "environ", None): + return self.environ["REMOTE_ADDR"] # type: ignore + + if not self.client_address: + return "" + + return self.client_address[0] + + def port_integer(self) -> int: + return self.client_address[1] + + # Escape control characters. This is defined (but private) in Python 3.12. + _control_char_table = str.maketrans( + {c: rf"\x{c:02x}" for c in [*range(0x20), *range(0x7F, 0xA0)]} + ) + _control_char_table[ord("\\")] = r"\\" + + def log_request(self, code: int | str = "-", size: int | str = "-") -> None: + try: + path = uri_to_iri(self.path) + msg = f"{self.command} {path} {self.request_version}" + except AttributeError: + # path isn't set if the requestline was bad + msg = self.requestline + + # Escape control characters that may be in the decoded path. + msg = msg.translate(self._control_char_table) + code = str(code) + + if code[0] == "1": # 1xx - Informational + msg = _ansi_style(msg, "bold") + elif code == "200": # 2xx - Success + pass + elif code == "304": # 304 - Resource Not Modified + msg = _ansi_style(msg, "cyan") + elif code[0] == "3": # 3xx - Redirection + msg = _ansi_style(msg, "green") + elif code == "404": # 404 - Resource Not Found + msg = _ansi_style(msg, "yellow") + elif code[0] == "4": # 4xx - Client Error + msg = _ansi_style(msg, "bold", "red") + else: # 5xx, or any other response + msg = _ansi_style(msg, "bold", "magenta") + + self.log("info", '"%s" %s %s', msg, code, size) + + def log_error(self, format: str, *args: t.Any) -> None: + self.log("error", format, *args) + + def log_message(self, format: str, *args: t.Any) -> None: + self.log("info", format, *args) + + def log(self, type: str, message: str, *args: t.Any) -> None: + # an IPv6 scoped address contains "%" which breaks logging + address_string = self.address_string().replace("%", "%%") + _log( + type, + f"{address_string} - - [{self.log_date_time_string()}] {message}\n", + *args, + ) + + +def _ansi_style(value: str, *styles: str) -> str: + if not _log_add_style: + return value + + codes = { + "bold": 1, + "red": 31, + "green": 32, + "yellow": 33, + "magenta": 35, + "cyan": 36, + } + + for style in styles: + value = f"\x1b[{codes[style]}m{value}" + + return f"{value}\x1b[0m" + + +def generate_adhoc_ssl_pair( + cn: str | None = None, +) -> tuple[Certificate, RSAPrivateKeyWithSerialization]: + try: + from cryptography import x509 + from cryptography.hazmat.backends import default_backend + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.asymmetric import rsa + from cryptography.x509.oid import NameOID + except ImportError: + raise TypeError( + "Using ad-hoc certificates requires the cryptography library." + ) from None + + backend = default_backend() + pkey = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=backend + ) + + # pretty damn sure that this is not actually accepted by anyone + if cn is None: + cn = "*" + + subject = x509.Name( + [ + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Dummy Certificate"), + x509.NameAttribute(NameOID.COMMON_NAME, cn), + ] + ) + + backend = default_backend() + cert = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(subject) + .public_key(pkey.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(dt.now(timezone.utc)) + .not_valid_after(dt.now(timezone.utc) + timedelta(days=365)) + .add_extension(x509.ExtendedKeyUsage([x509.OID_SERVER_AUTH]), critical=False) + .add_extension( + x509.SubjectAlternativeName([x509.DNSName(cn), x509.DNSName(f"*.{cn}")]), + critical=False, + ) + .sign(pkey, hashes.SHA256(), backend) + ) + return cert, pkey + + +def make_ssl_devcert( + base_path: str, host: str | None = None, cn: str | None = None +) -> tuple[str, str]: + """Creates an SSL key for development. This should be used instead of + the ``'adhoc'`` key which generates a new cert on each server start. + It accepts a path for where it should store the key and cert and + either a host or CN. If a host is given it will use the CN + ``*.host/CN=host``. + + For more information see :func:`run_simple`. + + .. versionadded:: 0.9 + + :param base_path: the path to the certificate and key. The extension + ``.crt`` is added for the certificate, ``.key`` is + added for the key. + :param host: the name of the host. This can be used as an alternative + for the `cn`. + :param cn: the `CN` to use. + """ + + if host is not None: + cn = host + cert, pkey = generate_adhoc_ssl_pair(cn=cn) + + from cryptography.hazmat.primitives import serialization + + cert_file = f"{base_path}.crt" + pkey_file = f"{base_path}.key" + + with open(cert_file, "wb") as f: + f.write(cert.public_bytes(serialization.Encoding.PEM)) + with open(pkey_file, "wb") as f: + f.write( + pkey.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + return cert_file, pkey_file + + +def generate_adhoc_ssl_context() -> ssl.SSLContext: + """Generates an adhoc SSL context for the development server.""" + import atexit + import tempfile + + cert, pkey = generate_adhoc_ssl_pair() + + from cryptography.hazmat.primitives import serialization + + cert_handle, cert_file = tempfile.mkstemp() + pkey_handle, pkey_file = tempfile.mkstemp() + atexit.register(os.remove, pkey_file) + atexit.register(os.remove, cert_file) + + os.write(cert_handle, cert.public_bytes(serialization.Encoding.PEM)) + os.write( + pkey_handle, + pkey.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ), + ) + + os.close(cert_handle) + os.close(pkey_handle) + ctx = load_ssl_context(cert_file, pkey_file) + return ctx + + +def load_ssl_context( + cert_file: str, pkey_file: str | None = None, protocol: int | None = None +) -> ssl.SSLContext: + """Loads SSL context from cert/private key files and optional protocol. + Many parameters are directly taken from the API of + :py:class:`ssl.SSLContext`. + + :param cert_file: Path of the certificate to use. + :param pkey_file: Path of the private key to use. If not given, the key + will be obtained from the certificate file. + :param protocol: A ``PROTOCOL`` constant from the :mod:`ssl` module. + Defaults to :data:`ssl.PROTOCOL_TLS_SERVER`. + """ + if protocol is None: + protocol = ssl.PROTOCOL_TLS_SERVER + + ctx = ssl.SSLContext(protocol) + ctx.load_cert_chain(cert_file, pkey_file) + return ctx + + +def is_ssl_error(error: Exception | None = None) -> bool: + """Checks if the given error (or the current one) is an SSL error.""" + if error is None: + error = t.cast(Exception, sys.exc_info()[1]) + return isinstance(error, ssl.SSLError) + + +def select_address_family(host: str, port: int) -> socket.AddressFamily: + """Return ``AF_INET4``, ``AF_INET6``, or ``AF_UNIX`` depending on + the host and port.""" + if host.startswith("unix://"): + return socket.AF_UNIX + elif ":" in host and hasattr(socket, "AF_INET6"): + return socket.AF_INET6 + return socket.AF_INET + + +def get_sockaddr( + host: str, port: int, family: socket.AddressFamily +) -> tuple[str, int] | str: + """Return a fully qualified socket address that can be passed to + :func:`socket.bind`.""" + if family == af_unix: + # Absolute path avoids IDNA encoding error when path starts with dot. + return os.path.abspath(host.partition("://")[2]) + try: + res = socket.getaddrinfo( + host, port, family, socket.SOCK_STREAM, socket.IPPROTO_TCP + ) + except socket.gaierror: + return host, port + return res[0][4] # type: ignore + + +def get_interface_ip(family: socket.AddressFamily) -> str: + """Get the IP address of an external interface. Used when binding to + 0.0.0.0 or ::1 to show a more useful URL. + + :meta private: + """ + # arbitrary private address + host = "fd31:f903:5ab5:1::1" if family == socket.AF_INET6 else "10.253.155.219" + + with socket.socket(family, socket.SOCK_DGRAM) as s: + try: + s.connect((host, 58162)) + except OSError: + return "::1" if family == socket.AF_INET6 else "127.0.0.1" + + return s.getsockname()[0] # type: ignore + + +class BaseWSGIServer(HTTPServer): + """A WSGI server that that handles one request at a time. + + Use :func:`make_server` to create a server instance. + """ + + multithread = False + multiprocess = False + request_queue_size = LISTEN_QUEUE + allow_reuse_address = True + + def __init__( + self, + host: str, + port: int, + app: WSGIApplication, + handler: type[WSGIRequestHandler] | None = None, + passthrough_errors: bool = False, + ssl_context: _TSSLContextArg | None = None, + fd: int | None = None, + ) -> None: + if handler is None: + handler = WSGIRequestHandler + + # If the handler doesn't directly set a protocol version and + # thread or process workers are used, then allow chunked + # responses and keep-alive connections by enabling HTTP/1.1. + if "protocol_version" not in vars(handler) and ( + self.multithread or self.multiprocess + ): + handler.protocol_version = "HTTP/1.1" + + self.host = host + self.port = port + self.app = app + self.passthrough_errors = passthrough_errors + + self.address_family = address_family = select_address_family(host, port) + server_address = get_sockaddr(host, int(port), address_family) + + # Remove a leftover Unix socket file from a previous run. Don't + # remove a file that was set up by run_simple. + if address_family == af_unix and fd is None: + server_address = t.cast(str, server_address) + + if os.path.exists(server_address): + os.unlink(server_address) + + # Bind and activate will be handled manually, it should only + # happen if we're not using a socket that was already set up. + super().__init__( + server_address, # type: ignore[arg-type] + handler, + bind_and_activate=False, + ) + + if fd is None: + # No existing socket descriptor, do bind_and_activate=True. + try: + self.server_bind() + self.server_activate() + except OSError as e: + # Catch connection issues and show them without the traceback. Show + # extra instructions for address not found, and for macOS. + self.server_close() + print(e.strerror, file=sys.stderr) + + if e.errno == errno.EADDRINUSE: + print( + f"Port {port} is in use by another program. Either identify and" + " stop that program, or start the server with a different" + " port.", + file=sys.stderr, + ) + + if sys.platform == "darwin" and port == 5000: + print( + "On macOS, try searching for and disabling" + " 'AirPlay Receiver' in System Settings.", + file=sys.stderr, + ) + + sys.exit(1) + except BaseException: + self.server_close() + raise + else: + # TCPServer automatically opens a socket even if bind_and_activate is False. + # Close it to silence a ResourceWarning. + self.server_close() + + # Use the passed in socket directly. + self.socket = socket.fromfd(fd, address_family, socket.SOCK_STREAM) + self.server_address = self.socket.getsockname() + + if address_family != af_unix: + # If port was 0, this will record the bound port. + self.port = self.server_address[1] + + if ssl_context is not None: + if isinstance(ssl_context, tuple): + ssl_context = load_ssl_context(*ssl_context) + elif ssl_context == "adhoc": + ssl_context = generate_adhoc_ssl_context() + + self.socket = ssl_context.wrap_socket(self.socket, server_side=True) + self.ssl_context: ssl.SSLContext | None = ssl_context + else: + self.ssl_context = None + + import importlib.metadata + + self._server_version = f"Werkzeug/{importlib.metadata.version('werkzeug')}" + + def log(self, type: str, message: str, *args: t.Any) -> None: + _log(type, message, *args) + + def serve_forever(self, poll_interval: float = 0.5) -> None: + try: + super().serve_forever(poll_interval=poll_interval) + except KeyboardInterrupt: + pass + finally: + self.server_close() + + def handle_error( + self, request: t.Any, client_address: tuple[str, int] | str + ) -> None: + if self.passthrough_errors: + raise + + return super().handle_error(request, client_address) + + def log_startup(self) -> None: + """Show information about the address when starting the server.""" + dev_warning = ( + "WARNING: This is a development server. Do not use it in a production" + " deployment. Use a production WSGI server instead." + ) + dev_warning = _ansi_style(dev_warning, "bold", "red") + messages = [dev_warning] + + if self.address_family == af_unix: + messages.append(f" * Running on {self.host}") + else: + scheme = "http" if self.ssl_context is None else "https" + display_hostname = self.host + + if self.host in {"0.0.0.0", "::"}: + messages.append(f" * Running on all addresses ({self.host})") + + if self.host == "0.0.0.0": + localhost = "127.0.0.1" + display_hostname = get_interface_ip(socket.AF_INET) + else: + localhost = "[::1]" + display_hostname = get_interface_ip(socket.AF_INET6) + + messages.append(f" * Running on {scheme}://{localhost}:{self.port}") + + if ":" in display_hostname: + display_hostname = f"[{display_hostname}]" + + messages.append(f" * Running on {scheme}://{display_hostname}:{self.port}") + + _log("info", "\n".join(messages)) + + +class ThreadedWSGIServer(socketserver.ThreadingMixIn, BaseWSGIServer): + """A WSGI server that handles concurrent requests in separate + threads. + + Use :func:`make_server` to create a server instance. + """ + + multithread = True + daemon_threads = True + + +class ForkingWSGIServer(ForkingMixIn, BaseWSGIServer): + """A WSGI server that handles concurrent requests in separate forked + processes. + + Use :func:`make_server` to create a server instance. + """ + + multiprocess = True + + def __init__( + self, + host: str, + port: int, + app: WSGIApplication, + processes: int = 40, + handler: type[WSGIRequestHandler] | None = None, + passthrough_errors: bool = False, + ssl_context: _TSSLContextArg | None = None, + fd: int | None = None, + ) -> None: + if not can_fork: + raise ValueError("Your platform does not support forking.") + + super().__init__(host, port, app, handler, passthrough_errors, ssl_context, fd) + self.max_children = processes + + +def make_server( + host: str, + port: int, + app: WSGIApplication, + threaded: bool = False, + processes: int = 1, + request_handler: type[WSGIRequestHandler] | None = None, + passthrough_errors: bool = False, + ssl_context: _TSSLContextArg | None = None, + fd: int | None = None, +) -> BaseWSGIServer: + """Create an appropriate WSGI server instance based on the value of + ``threaded`` and ``processes``. + + This is called from :func:`run_simple`, but can be used separately + to have access to the server object, such as to run it in a separate + thread. + + See :func:`run_simple` for parameter docs. + """ + if threaded and processes > 1: + raise ValueError("Cannot have a multi-thread and multi-process server.") + + if threaded: + return ThreadedWSGIServer( + host, port, app, request_handler, passthrough_errors, ssl_context, fd=fd + ) + + if processes > 1: + return ForkingWSGIServer( + host, + port, + app, + processes, + request_handler, + passthrough_errors, + ssl_context, + fd=fd, + ) + + return BaseWSGIServer( + host, port, app, request_handler, passthrough_errors, ssl_context, fd=fd + ) + + +def is_running_from_reloader() -> bool: + """Check if the server is running as a subprocess within the + Werkzeug reloader. + + .. versionadded:: 0.10 + """ + return os.environ.get("WERKZEUG_RUN_MAIN") == "true" + + +def run_simple( + hostname: str, + port: int, + application: WSGIApplication, + use_reloader: bool = False, + use_debugger: bool = False, + use_evalex: bool = True, + extra_files: t.Iterable[str] | None = None, + exclude_patterns: t.Iterable[str] | None = None, + reloader_interval: int = 1, + reloader_type: str = "auto", + threaded: bool = False, + processes: int = 1, + request_handler: type[WSGIRequestHandler] | None = None, + static_files: dict[str, str | tuple[str, str]] | None = None, + passthrough_errors: bool = False, + ssl_context: _TSSLContextArg | None = None, +) -> None: + """Start a development server for a WSGI application. Various + optional features can be enabled. + + .. warning:: + + Do not use the development server when deploying to production. + It is intended for use only during local development. It is not + designed to be particularly efficient, stable, or secure. + + :param hostname: The host to bind to, for example ``'localhost'``. + Can be a domain, IPv4 or IPv6 address, or file path starting + with ``unix://`` for a Unix socket. + :param port: The port to bind to, for example ``8080``. Using ``0`` + tells the OS to pick a random free port. + :param application: The WSGI application to run. + :param use_reloader: Use a reloader process to restart the server + process when files are changed. + :param use_debugger: Use Werkzeug's debugger, which will show + formatted tracebacks on unhandled exceptions. + :param use_evalex: Make the debugger interactive. A Python terminal + can be opened for any frame in the traceback. Some protection is + provided by requiring a PIN, but this should never be enabled + on a publicly visible server. + :param extra_files: The reloader will watch these files for changes + in addition to Python modules. For example, watch a + configuration file. + :param exclude_patterns: The reloader will ignore changes to any + files matching these :mod:`fnmatch` patterns. For example, + ignore cache files. + :param reloader_interval: How often the reloader tries to check for + changes. + :param reloader_type: The reloader to use. The ``'stat'`` reloader + is built in, but may require significant CPU to watch files. The + ``'watchdog'`` reloader is much more efficient but requires + installing the ``watchdog`` package first. + :param threaded: Handle concurrent requests using threads. Cannot be + used with ``processes``. + :param processes: Handle concurrent requests using up to this number + of processes. Cannot be used with ``threaded``. + :param request_handler: Use a different + :class:`~BaseHTTPServer.BaseHTTPRequestHandler` subclass to + handle requests. + :param static_files: A dict mapping URL prefixes to directories to + serve static files from using + :class:`~werkzeug.middleware.SharedDataMiddleware`. + :param passthrough_errors: Don't catch unhandled exceptions at the + server level, let the server crash instead. If ``use_debugger`` + is enabled, the debugger will still catch such errors. + :param ssl_context: Configure TLS to serve over HTTPS. Can be an + :class:`ssl.SSLContext` object, a ``(cert_file, key_file)`` + tuple to create a typical context, or the string ``'adhoc'`` to + generate a temporary self-signed certificate. + + .. versionchanged:: 2.1 + Instructions are shown for dealing with an "address already in + use" error. + + .. versionchanged:: 2.1 + Running on ``0.0.0.0`` or ``::`` shows the loopback IP in + addition to a real IP. + + .. versionchanged:: 2.1 + The command-line interface was removed. + + .. versionchanged:: 2.0 + Running on ``0.0.0.0`` or ``::`` shows a real IP address that + was bound as well as a warning not to run the development server + in production. + + .. versionchanged:: 2.0 + The ``exclude_patterns`` parameter was added. + + .. versionchanged:: 0.15 + Bind to a Unix socket by passing a ``hostname`` that starts with + ``unix://``. + + .. versionchanged:: 0.10 + Improved the reloader and added support for changing the backend + through the ``reloader_type`` parameter. + + .. versionchanged:: 0.9 + A command-line interface was added. + + .. versionchanged:: 0.8 + ``ssl_context`` can be a tuple of paths to the certificate and + private key files. + + .. versionchanged:: 0.6 + The ``ssl_context`` parameter was added. + + .. versionchanged:: 0.5 + The ``static_files`` and ``passthrough_errors`` parameters were + added. + """ + if not isinstance(port, int): + raise TypeError("port must be an integer") + + if static_files: + from .middleware.shared_data import SharedDataMiddleware + + application = SharedDataMiddleware(application, static_files) + + if use_debugger: + from .debug import DebuggedApplication + + application = DebuggedApplication(application, evalex=use_evalex) + # Allow the specified hostname to use the debugger, in addition to + # localhost domains. + application.trusted_hosts.append(hostname) + + if not is_running_from_reloader(): + fd = None + else: + fd = int(os.environ["WERKZEUG_SERVER_FD"]) + + srv = make_server( + hostname, + port, + application, + threaded, + processes, + request_handler, + passthrough_errors, + ssl_context, + fd=fd, + ) + srv.socket.set_inheritable(True) + os.environ["WERKZEUG_SERVER_FD"] = str(srv.fileno()) + + if not is_running_from_reloader(): + srv.log_startup() + _log("info", _ansi_style("Press CTRL+C to quit", "yellow")) + + if use_reloader: + from ._reloader import run_with_reloader + + try: + run_with_reloader( + srv.serve_forever, + extra_files=extra_files, + exclude_patterns=exclude_patterns, + interval=reloader_interval, + reloader_type=reloader_type, + ) + finally: + srv.server_close() + else: + srv.serve_forever() diff --git a/lib/python3.12/site-packages/werkzeug/test.py b/lib/python3.12/site-packages/werkzeug/test.py new file mode 100644 index 0000000000000000000000000000000000000000..58b4ce7faf216dc3e401513f01855005cae9295a --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/test.py @@ -0,0 +1,1464 @@ +from __future__ import annotations + +import dataclasses +import mimetypes +import sys +import typing as t +from collections import defaultdict +from datetime import datetime +from io import BytesIO +from itertools import chain +from random import random +from tempfile import TemporaryFile +from time import time +from urllib.parse import unquote +from urllib.parse import urlsplit +from urllib.parse import urlunsplit + +from ._internal import _get_environ +from ._internal import _wsgi_decoding_dance +from ._internal import _wsgi_encoding_dance +from .datastructures import Authorization +from .datastructures import CallbackDict +from .datastructures import CombinedMultiDict +from .datastructures import EnvironHeaders +from .datastructures import FileMultiDict +from .datastructures import Headers +from .datastructures import MultiDict +from .http import dump_cookie +from .http import dump_options_header +from .http import parse_cookie +from .http import parse_date +from .http import parse_options_header +from .sansio.multipart import Data +from .sansio.multipart import Epilogue +from .sansio.multipart import Field +from .sansio.multipart import File +from .sansio.multipart import MultipartEncoder +from .sansio.multipart import Preamble +from .urls import _urlencode +from .urls import iri_to_uri +from .utils import cached_property +from .utils import get_content_type +from .wrappers.request import Request +from .wrappers.response import Response +from .wsgi import ClosingIterator +from .wsgi import get_current_url + +if t.TYPE_CHECKING: + import typing_extensions as te + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +def stream_encode_multipart( + data: t.Mapping[str, t.Any], + use_tempfile: bool = True, + threshold: int = 1024 * 500, + boundary: str | None = None, +) -> tuple[t.IO[bytes], int, str]: + """Encode a dict of values (either strings or file descriptors or + :class:`FileStorage` objects.) into a multipart encoded string stored + in a file descriptor. + + .. versionchanged:: 3.0 + The ``charset`` parameter was removed. + """ + if boundary is None: + boundary = f"---------------WerkzeugFormPart_{time()}{random()}" + + stream: t.IO[bytes] = BytesIO() + total_length = 0 + on_disk = False + write_binary: t.Callable[[bytes], int] + + if use_tempfile: + + def write_binary(s: bytes) -> int: + nonlocal stream, total_length, on_disk + + if on_disk: + return stream.write(s) + else: + length = len(s) + + if length + total_length <= threshold: + stream.write(s) + else: + new_stream = t.cast(t.IO[bytes], TemporaryFile("wb+")) + new_stream.write(stream.getvalue()) # type: ignore + new_stream.write(s) + stream = new_stream + on_disk = True + + total_length += length + return length + + else: + write_binary = stream.write + + encoder = MultipartEncoder(boundary.encode()) + write_binary(encoder.send_event(Preamble(data=b""))) + for key, value in _iter_data(data): + reader = getattr(value, "read", None) + if reader is not None: + filename = getattr(value, "filename", getattr(value, "name", None)) + content_type = getattr(value, "content_type", None) + if content_type is None: + content_type = ( + filename + and mimetypes.guess_type(filename)[0] + or "application/octet-stream" + ) + headers = value.headers + headers.update([("Content-Type", content_type)]) + if filename is None: + write_binary(encoder.send_event(Field(name=key, headers=headers))) + else: + write_binary( + encoder.send_event( + File(name=key, filename=filename, headers=headers) + ) + ) + while True: + chunk = reader(16384) + + if not chunk: + write_binary(encoder.send_event(Data(data=chunk, more_data=False))) + break + + write_binary(encoder.send_event(Data(data=chunk, more_data=True))) + else: + if not isinstance(value, str): + value = str(value) + write_binary(encoder.send_event(Field(name=key, headers=Headers()))) + write_binary(encoder.send_event(Data(data=value.encode(), more_data=False))) + + write_binary(encoder.send_event(Epilogue(data=b""))) + + length = stream.tell() + stream.seek(0) + return stream, length, boundary + + +def encode_multipart( + values: t.Mapping[str, t.Any], boundary: str | None = None +) -> tuple[str, bytes]: + """Like `stream_encode_multipart` but returns a tuple in the form + (``boundary``, ``data``) where data is bytes. + + .. versionchanged:: 3.0 + The ``charset`` parameter was removed. + """ + stream, length, boundary = stream_encode_multipart( + values, use_tempfile=False, boundary=boundary + ) + return boundary, stream.read() + + +def _iter_data(data: t.Mapping[str, t.Any]) -> t.Iterator[tuple[str, t.Any]]: + """Iterate over a mapping that might have a list of values, yielding + all key, value pairs. Almost like iter_multi_items but only allows + lists, not tuples, of values so tuples can be used for files. + """ + if isinstance(data, MultiDict): + yield from data.items(multi=True) + else: + for key, value in data.items(): + if isinstance(value, list): + for v in value: + yield key, v + else: + yield key, value + + +_TAnyMultiDict = t.TypeVar("_TAnyMultiDict", bound="MultiDict[t.Any, t.Any]") + + +class EnvironBuilder: + """This class can be used to conveniently create a WSGI environment + for testing purposes. It can be used to quickly create WSGI environments + or request objects from arbitrary data. + + The signature of this class is also used in some other places as of + Werkzeug 0.5 (:func:`create_environ`, :meth:`Response.from_values`, + :meth:`Client.open`). Because of this most of the functionality is + available through the constructor alone. + + Files and regular form data can be manipulated independently of each + other with the :attr:`form` and :attr:`files` attributes, but are + passed with the same argument to the constructor: `data`. + + `data` can be any of these values: + + - a `str` or `bytes` object: The object is converted into an + :attr:`input_stream`, the :attr:`content_length` is set and you have to + provide a :attr:`content_type`. + - a `dict` or :class:`MultiDict`: The keys have to be strings. The values + have to be either any of the following objects, or a list of any of the + following objects: + + - a :class:`file`-like object: These are converted into + :class:`FileStorage` objects automatically. + - a `tuple`: The :meth:`~FileMultiDict.add_file` method is called + with the key and the unpacked `tuple` items as positional + arguments. + - a `str`: The string is set as form data for the associated key. + - a file-like object: The object content is loaded in memory and then + handled like a regular `str` or a `bytes`. + + :param path: the path of the request. In the WSGI environment this will + end up as `PATH_INFO`. If the `query_string` is not defined + and there is a question mark in the `path` everything after + it is used as query string. + :param base_url: the base URL is a URL that is used to extract the WSGI + URL scheme, host (server name + server port) and the + script root (`SCRIPT_NAME`). + :param query_string: an optional string or dict with URL parameters. + :param method: the HTTP method to use, defaults to `GET`. + :param input_stream: an optional input stream. Do not specify this and + `data`. As soon as an input stream is set you can't + modify :attr:`args` and :attr:`files` unless you + set the :attr:`input_stream` to `None` again. + :param content_type: The content type for the request. As of 0.5 you + don't have to provide this when specifying files + and form data via `data`. + :param content_length: The content length for the request. You don't + have to specify this when providing data via + `data`. + :param errors_stream: an optional error stream that is used for + `wsgi.errors`. Defaults to :data:`stderr`. + :param multithread: controls `wsgi.multithread`. Defaults to `False`. + :param multiprocess: controls `wsgi.multiprocess`. Defaults to `False`. + :param run_once: controls `wsgi.run_once`. Defaults to `False`. + :param headers: an optional list or :class:`Headers` object of headers. + :param data: a string or dict of form data or a file-object. + See explanation above. + :param json: An object to be serialized and assigned to ``data``. + Defaults the content type to ``"application/json"``. + Serialized with the function assigned to :attr:`json_dumps`. + :param environ_base: an optional dict of environment defaults. + :param environ_overrides: an optional dict of environment overrides. + :param auth: An authorization object to use for the + ``Authorization`` header value. A ``(username, password)`` tuple + is a shortcut for ``Basic`` authorization. + + .. versionchanged:: 3.0 + The ``charset`` parameter was removed. + + .. versionchanged:: 2.1 + ``CONTENT_TYPE`` and ``CONTENT_LENGTH`` are not duplicated as + header keys in the environ. + + .. versionchanged:: 2.0 + ``REQUEST_URI`` and ``RAW_URI`` is the full raw URI including + the query string, not only the path. + + .. versionchanged:: 2.0 + The default :attr:`request_class` is ``Request`` instead of + ``BaseRequest``. + + .. versionadded:: 2.0 + Added the ``auth`` parameter. + + .. versionadded:: 0.15 + The ``json`` param and :meth:`json_dumps` method. + + .. versionadded:: 0.15 + The environ has keys ``REQUEST_URI`` and ``RAW_URI`` containing + the path before percent-decoding. This is not part of the WSGI + PEP, but many WSGI servers include it. + + .. versionchanged:: 0.6 + ``path`` and ``base_url`` can now be unicode strings that are + encoded with :func:`iri_to_uri`. + """ + + #: the server protocol to use. defaults to HTTP/1.1 + server_protocol = "HTTP/1.1" + + #: the wsgi version to use. defaults to (1, 0) + wsgi_version = (1, 0) + + #: The default request class used by :meth:`get_request`. + request_class = Request + + import json + + #: The serialization function used when ``json`` is passed. + json_dumps = staticmethod(json.dumps) + del json + + _args: MultiDict[str, str] | None + _query_string: str | None + _input_stream: t.IO[bytes] | None + _form: MultiDict[str, str] | None + _files: FileMultiDict | None + + def __init__( + self, + path: str = "/", + base_url: str | None = None, + query_string: t.Mapping[str, str] | str | None = None, + method: str = "GET", + input_stream: t.IO[bytes] | None = None, + content_type: str | None = None, + content_length: int | None = None, + errors_stream: t.IO[str] | None = None, + multithread: bool = False, + multiprocess: bool = False, + run_once: bool = False, + headers: Headers | t.Iterable[tuple[str, str]] | None = None, + data: None | (t.IO[bytes] | str | bytes | t.Mapping[str, t.Any]) = None, + environ_base: t.Mapping[str, t.Any] | None = None, + environ_overrides: t.Mapping[str, t.Any] | None = None, + mimetype: str | None = None, + json: t.Mapping[str, t.Any] | None = None, + auth: Authorization | tuple[str, str] | None = None, + ) -> None: + if query_string is not None and "?" in path: + raise ValueError("Query string is defined in the path and as an argument") + request_uri = urlsplit(path) + if query_string is None and "?" in path: + query_string = request_uri.query + + self.path = iri_to_uri(request_uri.path) + self.request_uri = path + if base_url is not None: + base_url = iri_to_uri(base_url) + self.base_url = base_url + if isinstance(query_string, str): + self.query_string = query_string + else: + if query_string is None: + query_string = MultiDict() + elif not isinstance(query_string, MultiDict): + query_string = MultiDict(query_string) + self.args = query_string + self.method = method + if headers is None: + headers = Headers() + elif not isinstance(headers, Headers): + headers = Headers(headers) + self.headers = headers + if content_type is not None: + self.content_type = content_type + if errors_stream is None: + errors_stream = sys.stderr + self.errors_stream = errors_stream + self.multithread = multithread + self.multiprocess = multiprocess + self.run_once = run_once + self.environ_base = environ_base + self.environ_overrides = environ_overrides + self.input_stream = input_stream + self.content_length = content_length + self.closed = False + + if auth is not None: + if isinstance(auth, tuple): + auth = Authorization( + "basic", {"username": auth[0], "password": auth[1]} + ) + + self.headers.set("Authorization", auth.to_header()) + + if json is not None: + if data is not None: + raise TypeError("can't provide both json and data") + + data = self.json_dumps(json) + + if self.content_type is None: + self.content_type = "application/json" + + if data: + if input_stream is not None: + raise TypeError("can't provide input stream and data") + if hasattr(data, "read"): + data = data.read() + if isinstance(data, str): + data = data.encode() + if isinstance(data, bytes): + self.input_stream = BytesIO(data) + if self.content_length is None: + self.content_length = len(data) + else: + for key, value in _iter_data(data): + if isinstance(value, (tuple, dict)) or hasattr(value, "read"): + self._add_file_from_data(key, value) + else: + self.form.setlistdefault(key).append(value) + + if mimetype is not None: + self.mimetype = mimetype + + @classmethod + def from_environ(cls, environ: WSGIEnvironment, **kwargs: t.Any) -> EnvironBuilder: + """Turn an environ dict back into a builder. Any extra kwargs + override the args extracted from the environ. + + .. versionchanged:: 2.0 + Path and query values are passed through the WSGI decoding + dance to avoid double encoding. + + .. versionadded:: 0.15 + """ + headers = Headers(EnvironHeaders(environ)) + out = { + "path": _wsgi_decoding_dance(environ["PATH_INFO"]), + "base_url": cls._make_base_url( + environ["wsgi.url_scheme"], + headers.pop("Host"), + _wsgi_decoding_dance(environ["SCRIPT_NAME"]), + ), + "query_string": _wsgi_decoding_dance(environ["QUERY_STRING"]), + "method": environ["REQUEST_METHOD"], + "input_stream": environ["wsgi.input"], + "content_type": headers.pop("Content-Type", None), + "content_length": headers.pop("Content-Length", None), + "errors_stream": environ["wsgi.errors"], + "multithread": environ["wsgi.multithread"], + "multiprocess": environ["wsgi.multiprocess"], + "run_once": environ["wsgi.run_once"], + "headers": headers, + } + out.update(kwargs) + return cls(**out) + + def _add_file_from_data( + self, + key: str, + value: (t.IO[bytes] | tuple[t.IO[bytes], str] | tuple[t.IO[bytes], str, str]), + ) -> None: + """Called in the EnvironBuilder to add files from the data dict.""" + if isinstance(value, tuple): + self.files.add_file(key, *value) + else: + self.files.add_file(key, value) + + @staticmethod + def _make_base_url(scheme: str, host: str, script_root: str) -> str: + return urlunsplit((scheme, host, script_root, "", "")).rstrip("/") + "/" + + @property + def base_url(self) -> str: + """The base URL is used to extract the URL scheme, host name, + port, and root path. + """ + return self._make_base_url(self.url_scheme, self.host, self.script_root) + + @base_url.setter + def base_url(self, value: str | None) -> None: + if value is None: + scheme = "http" + netloc = "localhost" + script_root = "" + else: + scheme, netloc, script_root, qs, anchor = urlsplit(value) + if qs or anchor: + raise ValueError("base url must not contain a query string or fragment") + self.script_root = script_root.rstrip("/") + self.host = netloc + self.url_scheme = scheme + + @property + def content_type(self) -> str | None: + """The content type for the request. Reflected from and to + the :attr:`headers`. Do not set if you set :attr:`files` or + :attr:`form` for auto detection. + """ + ct = self.headers.get("Content-Type") + if ct is None and not self._input_stream: + if self._files: + return "multipart/form-data" + if self._form: + return "application/x-www-form-urlencoded" + return None + return ct + + @content_type.setter + def content_type(self, value: str | None) -> None: + if value is None: + self.headers.pop("Content-Type", None) + else: + self.headers["Content-Type"] = value + + @property + def mimetype(self) -> str | None: + """The mimetype (content type without charset etc.) + + .. versionadded:: 0.14 + """ + ct = self.content_type + return ct.split(";")[0].strip() if ct else None + + @mimetype.setter + def mimetype(self, value: str) -> None: + self.content_type = get_content_type(value, "utf-8") + + @property + def mimetype_params(self) -> t.Mapping[str, str]: + """The mimetype parameters as dict. For example if the + content type is ``text/html; charset=utf-8`` the params would be + ``{'charset': 'utf-8'}``. + + .. versionadded:: 0.14 + """ + + def on_update(d: CallbackDict[str, str]) -> None: + self.headers["Content-Type"] = dump_options_header(self.mimetype, d) + + d = parse_options_header(self.headers.get("content-type", ""))[1] + return CallbackDict(d, on_update) + + @property + def content_length(self) -> int | None: + """The content length as integer. Reflected from and to the + :attr:`headers`. Do not set if you set :attr:`files` or + :attr:`form` for auto detection. + """ + return self.headers.get("Content-Length", type=int) + + @content_length.setter + def content_length(self, value: int | None) -> None: + if value is None: + self.headers.pop("Content-Length", None) + else: + self.headers["Content-Length"] = str(value) + + def _get_form(self, name: str, storage: type[_TAnyMultiDict]) -> _TAnyMultiDict: + """Common behavior for getting the :attr:`form` and + :attr:`files` properties. + + :param name: Name of the internal cached attribute. + :param storage: Storage class used for the data. + """ + if self.input_stream is not None: + raise AttributeError("an input stream is defined") + + rv = getattr(self, name) + + if rv is None: + rv = storage() + setattr(self, name, rv) + + return rv # type: ignore + + def _set_form(self, name: str, value: MultiDict[str, t.Any]) -> None: + """Common behavior for setting the :attr:`form` and + :attr:`files` properties. + + :param name: Name of the internal cached attribute. + :param value: Value to assign to the attribute. + """ + self._input_stream = None + setattr(self, name, value) + + @property + def form(self) -> MultiDict[str, str]: + """A :class:`MultiDict` of form values.""" + return self._get_form("_form", MultiDict) + + @form.setter + def form(self, value: MultiDict[str, str]) -> None: + self._set_form("_form", value) + + @property + def files(self) -> FileMultiDict: + """A :class:`FileMultiDict` of uploaded files. Use + :meth:`~FileMultiDict.add_file` to add new files. + """ + return self._get_form("_files", FileMultiDict) + + @files.setter + def files(self, value: FileMultiDict) -> None: + self._set_form("_files", value) + + @property + def input_stream(self) -> t.IO[bytes] | None: + """An optional input stream. This is mutually exclusive with + setting :attr:`form` and :attr:`files`, setting it will clear + those. Do not provide this if the method is not ``POST`` or + another method that has a body. + """ + return self._input_stream + + @input_stream.setter + def input_stream(self, value: t.IO[bytes] | None) -> None: + self._input_stream = value + self._form = None + self._files = None + + @property + def query_string(self) -> str: + """The query string. If you set this to a string + :attr:`args` will no longer be available. + """ + if self._query_string is None: + if self._args is not None: + return _urlencode(self._args) + return "" + return self._query_string + + @query_string.setter + def query_string(self, value: str | None) -> None: + self._query_string = value + self._args = None + + @property + def args(self) -> MultiDict[str, str]: + """The URL arguments as :class:`MultiDict`.""" + if self._query_string is not None: + raise AttributeError("a query string is defined") + if self._args is None: + self._args = MultiDict() + return self._args + + @args.setter + def args(self, value: MultiDict[str, str] | None) -> None: + self._query_string = None + self._args = value + + @property + def server_name(self) -> str: + """The server name (read-only, use :attr:`host` to set)""" + return self.host.split(":", 1)[0] + + @property + def server_port(self) -> int: + """The server port as integer (read-only, use :attr:`host` to set)""" + pieces = self.host.split(":", 1) + + if len(pieces) == 2: + try: + return int(pieces[1]) + except ValueError: + pass + + if self.url_scheme == "https": + return 443 + return 80 + + def __del__(self) -> None: + try: + self.close() + except Exception: + pass + + def close(self) -> None: + """Closes all files. If you put real :class:`file` objects into the + :attr:`files` dict you can call this method to automatically close + them all in one go. + """ + if self.closed: + return + try: + files = self.files.values() + except AttributeError: + files = () + for f in files: + try: + f.close() + except Exception: + pass + self.closed = True + + def get_environ(self) -> WSGIEnvironment: + """Return the built environ. + + .. versionchanged:: 0.15 + The content type and length headers are set based on + input stream detection. Previously this only set the WSGI + keys. + """ + input_stream = self.input_stream + content_length = self.content_length + + mimetype = self.mimetype + content_type = self.content_type + + if input_stream is not None: + start_pos = input_stream.tell() + input_stream.seek(0, 2) + end_pos = input_stream.tell() + input_stream.seek(start_pos) + content_length = end_pos - start_pos + elif mimetype == "multipart/form-data": + input_stream, content_length, boundary = stream_encode_multipart( + CombinedMultiDict([self.form, self.files]) + ) + content_type = f'{mimetype}; boundary="{boundary}"' + elif mimetype == "application/x-www-form-urlencoded": + form_encoded = _urlencode(self.form).encode("ascii") + content_length = len(form_encoded) + input_stream = BytesIO(form_encoded) + else: + input_stream = BytesIO() + + result: WSGIEnvironment = {} + if self.environ_base: + result.update(self.environ_base) + + def _path_encode(x: str) -> str: + return _wsgi_encoding_dance(unquote(x)) + + raw_uri = _wsgi_encoding_dance(self.request_uri) + result.update( + { + "REQUEST_METHOD": self.method, + "SCRIPT_NAME": _path_encode(self.script_root), + "PATH_INFO": _path_encode(self.path), + "QUERY_STRING": _wsgi_encoding_dance(self.query_string), + # Non-standard, added by mod_wsgi, uWSGI + "REQUEST_URI": raw_uri, + # Non-standard, added by gunicorn + "RAW_URI": raw_uri, + "SERVER_NAME": self.server_name, + "SERVER_PORT": str(self.server_port), + "HTTP_HOST": self.host, + "SERVER_PROTOCOL": self.server_protocol, + "wsgi.version": self.wsgi_version, + "wsgi.url_scheme": self.url_scheme, + "wsgi.input": input_stream, + "wsgi.errors": self.errors_stream, + "wsgi.multithread": self.multithread, + "wsgi.multiprocess": self.multiprocess, + "wsgi.run_once": self.run_once, + } + ) + + headers = self.headers.copy() + # Don't send these as headers, they're part of the environ. + headers.remove("Content-Type") + headers.remove("Content-Length") + + if content_type is not None: + result["CONTENT_TYPE"] = content_type + + if content_length is not None: + result["CONTENT_LENGTH"] = str(content_length) + + combined_headers = defaultdict(list) + + for key, value in headers.to_wsgi_list(): + combined_headers[f"HTTP_{key.upper().replace('-', '_')}"].append(value) + + for key, values in combined_headers.items(): + result[key] = ", ".join(values) + + if self.environ_overrides: + result.update(self.environ_overrides) + + return result + + def get_request(self, cls: type[Request] | None = None) -> Request: + """Returns a request with the data. If the request class is not + specified :attr:`request_class` is used. + + :param cls: The request wrapper to use. + """ + if cls is None: + cls = self.request_class + + return cls(self.get_environ()) + + +class ClientRedirectError(Exception): + """If a redirect loop is detected when using follow_redirects=True with + the :cls:`Client`, then this exception is raised. + """ + + +class Client: + """Simulate sending requests to a WSGI application without running a WSGI or HTTP + server. + + :param application: The WSGI application to make requests to. + :param response_wrapper: A :class:`.Response` class to wrap response data with. + Defaults to :class:`.TestResponse`. If it's not a subclass of ``TestResponse``, + one will be created. + :param use_cookies: Persist cookies from ``Set-Cookie`` response headers to the + ``Cookie`` header in subsequent requests. Domain and path matching is supported, + but other cookie parameters are ignored. + :param allow_subdomain_redirects: Allow requests to follow redirects to subdomains. + Enable this if the application handles subdomains and redirects between them. + + .. versionchanged:: 2.3 + Simplify cookie implementation, support domain and path matching. + + .. versionchanged:: 2.1 + All data is available as properties on the returned response object. The + response cannot be returned as a tuple. + + .. versionchanged:: 2.0 + ``response_wrapper`` is always a subclass of :class:``TestResponse``. + + .. versionchanged:: 0.5 + Added the ``use_cookies`` parameter. + """ + + def __init__( + self, + application: WSGIApplication, + response_wrapper: type[Response] | None = None, + use_cookies: bool = True, + allow_subdomain_redirects: bool = False, + ) -> None: + self.application = application + + if response_wrapper in {None, Response}: + response_wrapper = TestResponse + elif response_wrapper is not None and not issubclass( + response_wrapper, TestResponse + ): + response_wrapper = type( + "WrapperTestResponse", + (TestResponse, response_wrapper), + {}, + ) + + self.response_wrapper = t.cast(type["TestResponse"], response_wrapper) + + if use_cookies: + self._cookies: dict[tuple[str, str, str], Cookie] | None = {} + else: + self._cookies = None + + self.allow_subdomain_redirects = allow_subdomain_redirects + + def get_cookie( + self, key: str, domain: str = "localhost", path: str = "/" + ) -> Cookie | None: + """Return a :class:`.Cookie` if it exists. Cookies are uniquely identified by + ``(domain, path, key)``. + + :param key: The decoded form of the key for the cookie. + :param domain: The domain the cookie was set for. + :param path: The path the cookie was set for. + + .. versionadded:: 2.3 + """ + if self._cookies is None: + raise TypeError( + "Cookies are disabled. Create a client with 'use_cookies=True'." + ) + + return self._cookies.get((domain, path, key)) + + def set_cookie( + self, + key: str, + value: str = "", + *, + domain: str = "localhost", + origin_only: bool = True, + path: str = "/", + **kwargs: t.Any, + ) -> None: + """Set a cookie to be sent in subsequent requests. + + This is a convenience to skip making a test request to a route that would set + the cookie. To test the cookie, make a test request to a route that uses the + cookie value. + + The client uses ``domain``, ``origin_only``, and ``path`` to determine which + cookies to send with a request. It does not use other cookie parameters that + browsers use, since they're not applicable in tests. + + :param key: The key part of the cookie. + :param value: The value part of the cookie. + :param domain: Send this cookie with requests that match this domain. If + ``origin_only`` is true, it must be an exact match, otherwise it may be a + suffix match. + :param origin_only: Whether the domain must be an exact match to the request. + :param path: Send this cookie with requests that match this path either exactly + or as a prefix. + :param kwargs: Passed to :func:`.dump_cookie`. + + .. versionchanged:: 3.0 + The parameter ``server_name`` is removed. The first parameter is + ``key``. Use the ``domain`` and ``origin_only`` parameters instead. + + .. versionchanged:: 2.3 + The ``origin_only`` parameter was added. + + .. versionchanged:: 2.3 + The ``domain`` parameter defaults to ``localhost``. + """ + if self._cookies is None: + raise TypeError( + "Cookies are disabled. Create a client with 'use_cookies=True'." + ) + + cookie = Cookie._from_response_header( + domain, "/", dump_cookie(key, value, domain=domain, path=path, **kwargs) + ) + cookie.origin_only = origin_only + + if cookie._should_delete: + self._cookies.pop(cookie._storage_key, None) + else: + self._cookies[cookie._storage_key] = cookie + + def delete_cookie( + self, + key: str, + *, + domain: str = "localhost", + path: str = "/", + ) -> None: + """Delete a cookie if it exists. Cookies are uniquely identified by + ``(domain, path, key)``. + + :param key: The decoded form of the key for the cookie. + :param domain: The domain the cookie was set for. + :param path: The path the cookie was set for. + + .. versionchanged:: 3.0 + The ``server_name`` parameter is removed. The first parameter is + ``key``. Use the ``domain`` parameter instead. + + .. versionchanged:: 3.0 + The ``secure``, ``httponly`` and ``samesite`` parameters are removed. + + .. versionchanged:: 2.3 + The ``domain`` parameter defaults to ``localhost``. + """ + if self._cookies is None: + raise TypeError( + "Cookies are disabled. Create a client with 'use_cookies=True'." + ) + + self._cookies.pop((domain, path, key), None) + + def _add_cookies_to_wsgi(self, environ: WSGIEnvironment) -> None: + """If cookies are enabled, set the ``Cookie`` header in the environ to the + cookies that are applicable to the request host and path. + + :meta private: + + .. versionadded:: 2.3 + """ + if self._cookies is None: + return + + url = urlsplit(get_current_url(environ)) + server_name = url.hostname or "localhost" + value = "; ".join( + c._to_request_header() + for c in self._cookies.values() + if c._matches_request(server_name, url.path) + ) + + if value: + environ["HTTP_COOKIE"] = value + else: + environ.pop("HTTP_COOKIE", None) + + def _update_cookies_from_response( + self, server_name: str, path: str, headers: list[str] + ) -> None: + """If cookies are enabled, update the stored cookies from any ``Set-Cookie`` + headers in the response. + + :meta private: + + .. versionadded:: 2.3 + """ + if self._cookies is None: + return + + for header in headers: + cookie = Cookie._from_response_header(server_name, path, header) + + if cookie._should_delete: + self._cookies.pop(cookie._storage_key, None) + else: + self._cookies[cookie._storage_key] = cookie + + def run_wsgi_app( + self, environ: WSGIEnvironment, buffered: bool = False + ) -> tuple[t.Iterable[bytes], str, Headers]: + """Runs the wrapped WSGI app with the given environment. + + :meta private: + """ + self._add_cookies_to_wsgi(environ) + rv = run_wsgi_app(self.application, environ, buffered=buffered) + url = urlsplit(get_current_url(environ)) + self._update_cookies_from_response( + url.hostname or "localhost", url.path, rv[2].getlist("Set-Cookie") + ) + return rv + + def resolve_redirect( + self, response: TestResponse, buffered: bool = False + ) -> TestResponse: + """Perform a new request to the location given by the redirect + response to the previous request. + + :meta private: + """ + scheme, netloc, path, qs, anchor = urlsplit(response.location) + builder = EnvironBuilder.from_environ( + response.request.environ, path=path, query_string=qs + ) + + to_name_parts = netloc.split(":", 1)[0].split(".") + from_name_parts = builder.server_name.split(".") + + if to_name_parts != [""]: + # The new location has a host, use it for the base URL. + builder.url_scheme = scheme + builder.host = netloc + else: + # A local redirect with autocorrect_location_header=False + # doesn't have a host, so use the request's host. + to_name_parts = from_name_parts + + # Explain why a redirect to a different server name won't be followed. + if to_name_parts != from_name_parts: + if to_name_parts[-len(from_name_parts) :] == from_name_parts: + if not self.allow_subdomain_redirects: + raise RuntimeError("Following subdomain redirects is not enabled.") + else: + raise RuntimeError("Following external redirects is not supported.") + + path_parts = path.split("/") + root_parts = builder.script_root.split("/") + + if path_parts[: len(root_parts)] == root_parts: + # Strip the script root from the path. + builder.path = path[len(builder.script_root) :] + else: + # The new location is not under the script root, so use the + # whole path and clear the previous root. + builder.path = path + builder.script_root = "" + + # Only 307 and 308 preserve all of the original request. + if response.status_code not in {307, 308}: + # HEAD is preserved, everything else becomes GET. + if builder.method != "HEAD": + builder.method = "GET" + + # Clear the body and the headers that describe it. + + if builder.input_stream is not None: + builder.input_stream.close() + builder.input_stream = None + + builder.content_type = None + builder.content_length = None + builder.headers.pop("Transfer-Encoding", None) + + return self.open(builder, buffered=buffered) + + def open( + self, + *args: t.Any, + buffered: bool = False, + follow_redirects: bool = False, + **kwargs: t.Any, + ) -> TestResponse: + """Generate an environ dict from the given arguments, make a + request to the application using it, and return the response. + + :param args: Passed to :class:`EnvironBuilder` to create the + environ for the request. If a single arg is passed, it can + be an existing :class:`EnvironBuilder` or an environ dict. + :param buffered: Convert the iterator returned by the app into + a list. If the iterator has a ``close()`` method, it is + called automatically. + :param follow_redirects: Make additional requests to follow HTTP + redirects until a non-redirect status is returned. + :attr:`TestResponse.history` lists the intermediate + responses. + + .. versionchanged:: 2.1 + Removed the ``as_tuple`` parameter. + + .. versionchanged:: 2.0 + The request input stream is closed when calling + ``response.close()``. Input streams for redirects are + automatically closed. + + .. versionchanged:: 0.5 + If a dict is provided as file in the dict for the ``data`` + parameter the content type has to be called ``content_type`` + instead of ``mimetype``. This change was made for + consistency with :class:`werkzeug.FileWrapper`. + + .. versionchanged:: 0.5 + Added the ``follow_redirects`` parameter. + """ + request: Request | None = None + + if not kwargs and len(args) == 1: + arg = args[0] + + if isinstance(arg, EnvironBuilder): + request = arg.get_request() + elif isinstance(arg, dict): + request = EnvironBuilder.from_environ(arg).get_request() + elif isinstance(arg, Request): + request = arg + + if request is None: + builder = EnvironBuilder(*args, **kwargs) + + try: + request = builder.get_request() + finally: + builder.close() + + response_parts = self.run_wsgi_app(request.environ, buffered=buffered) + response = self.response_wrapper(*response_parts, request=request) + + redirects = set() + history: list[TestResponse] = [] + + if not follow_redirects: + return response + + while response.status_code in { + 301, + 302, + 303, + 305, + 307, + 308, + }: + # Exhaust intermediate response bodies to ensure middleware + # that returns an iterator runs any cleanup code. + if not buffered: + response.make_sequence() + response.close() + + new_redirect_entry = (response.location, response.status_code) + + if new_redirect_entry in redirects: + raise ClientRedirectError( + f"Loop detected: A {response.status_code} redirect" + f" to {response.location} was already made." + ) + + redirects.add(new_redirect_entry) + response.history = tuple(history) + history.append(response) + response = self.resolve_redirect(response, buffered=buffered) + else: + # This is the final request after redirects. + response.history = tuple(history) + # Close the input stream when closing the response, in case + # the input is an open temporary file. + response.call_on_close(request.input_stream.close) + return response + + def get(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``GET``.""" + kw["method"] = "GET" + return self.open(*args, **kw) + + def post(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``POST``.""" + kw["method"] = "POST" + return self.open(*args, **kw) + + def put(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``PUT``.""" + kw["method"] = "PUT" + return self.open(*args, **kw) + + def delete(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``DELETE``.""" + kw["method"] = "DELETE" + return self.open(*args, **kw) + + def patch(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``PATCH``.""" + kw["method"] = "PATCH" + return self.open(*args, **kw) + + def options(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``OPTIONS``.""" + kw["method"] = "OPTIONS" + return self.open(*args, **kw) + + def head(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``HEAD``.""" + kw["method"] = "HEAD" + return self.open(*args, **kw) + + def trace(self, *args: t.Any, **kw: t.Any) -> TestResponse: + """Call :meth:`open` with ``method`` set to ``TRACE``.""" + kw["method"] = "TRACE" + return self.open(*args, **kw) + + def __repr__(self) -> str: + return f"<{type(self).__name__} {self.application!r}>" + + +def create_environ(*args: t.Any, **kwargs: t.Any) -> WSGIEnvironment: + """Create a new WSGI environ dict based on the values passed. The first + parameter should be the path of the request which defaults to '/'. The + second one can either be an absolute path (in that case the host is + localhost:80) or a full path to the request with scheme, netloc port and + the path to the script. + + This accepts the same arguments as the :class:`EnvironBuilder` + constructor. + + .. versionchanged:: 0.5 + This function is now a thin wrapper over :class:`EnvironBuilder` which + was added in 0.5. The `headers`, `environ_base`, `environ_overrides` + and `charset` parameters were added. + """ + builder = EnvironBuilder(*args, **kwargs) + + try: + return builder.get_environ() + finally: + builder.close() + + +def run_wsgi_app( + app: WSGIApplication, environ: WSGIEnvironment, buffered: bool = False +) -> tuple[t.Iterable[bytes], str, Headers]: + """Return a tuple in the form (app_iter, status, headers) of the + application output. This works best if you pass it an application that + returns an iterator all the time. + + Sometimes applications may use the `write()` callable returned + by the `start_response` function. This tries to resolve such edge + cases automatically. But if you don't get the expected output you + should set `buffered` to `True` which enforces buffering. + + If passed an invalid WSGI application the behavior of this function is + undefined. Never pass non-conforming WSGI applications to this function. + + :param app: the application to execute. + :param buffered: set to `True` to enforce buffering. + :return: tuple in the form ``(app_iter, status, headers)`` + """ + # Copy environ to ensure any mutations by the app (ProxyFix, for + # example) don't affect subsequent requests (such as redirects). + environ = _get_environ(environ).copy() + status: str + response: tuple[str, list[tuple[str, str]]] | None = None + buffer: list[bytes] = [] + + def start_response(status, headers, exc_info=None): # type: ignore + nonlocal response + + if exc_info: + try: + raise exc_info[1].with_traceback(exc_info[2]) + finally: + exc_info = None + + response = (status, headers) + return buffer.append + + app_rv = app(environ, start_response) + close_func = getattr(app_rv, "close", None) + app_iter: t.Iterable[bytes] = iter(app_rv) + + # when buffering we emit the close call early and convert the + # application iterator into a regular list + if buffered: + try: + app_iter = list(app_iter) + finally: + if close_func is not None: + close_func() + + # otherwise we iterate the application iter until we have a response, chain + # the already received data with the already collected data and wrap it in + # a new `ClosingIterator` if we need to restore a `close` callable from the + # original return value. + else: + for item in app_iter: + buffer.append(item) + + if response is not None: + break + + if buffer: + app_iter = chain(buffer, app_iter) + + if close_func is not None and app_iter is not app_rv: + app_iter = ClosingIterator(app_iter, close_func) + + status, headers = response # type: ignore + return app_iter, status, Headers(headers) + + +class TestResponse(Response): + """:class:`~werkzeug.wrappers.Response` subclass that provides extra + information about requests made with the test :class:`Client`. + + Test client requests will always return an instance of this class. + If a custom response class is passed to the client, it is + subclassed along with this to support test information. + + If the test request included large files, or if the application is + serving a file, call :meth:`close` to close any open files and + prevent Python showing a ``ResourceWarning``. + + .. versionchanged:: 2.2 + Set the ``default_mimetype`` to None to prevent a mimetype being + assumed if missing. + + .. versionchanged:: 2.1 + Response instances cannot be treated as tuples. + + .. versionadded:: 2.0 + Test client methods always return instances of this class. + """ + + default_mimetype = None + # Don't assume a mimetype, instead use whatever the response provides + + request: Request + """A request object with the environ used to make the request that + resulted in this response. + """ + + history: tuple[TestResponse, ...] + """A list of intermediate responses. Populated when the test request + is made with ``follow_redirects`` enabled. + """ + + # Tell Pytest to ignore this, it's not a test class. + __test__ = False + + def __init__( + self, + response: t.Iterable[bytes], + status: str, + headers: Headers, + request: Request, + history: tuple[TestResponse] = (), # type: ignore + **kwargs: t.Any, + ) -> None: + super().__init__(response, status, headers, **kwargs) + self.request = request + self.history = history + self._compat_tuple = response, status, headers + + @cached_property + def text(self) -> str: + """The response data as text. A shortcut for + ``response.get_data(as_text=True)``. + + .. versionadded:: 2.1 + """ + return self.get_data(as_text=True) + + +@dataclasses.dataclass +class Cookie: + """A cookie key, value, and parameters. + + The class itself is not a public API. Its attributes are documented for inspection + with :meth:`.Client.get_cookie` only. + + .. versionadded:: 2.3 + """ + + key: str + """The cookie key, encoded as a client would see it.""" + + value: str + """The cookie key, encoded as a client would see it.""" + + decoded_key: str + """The cookie key, decoded as the application would set and see it.""" + + decoded_value: str + """The cookie value, decoded as the application would set and see it.""" + + expires: datetime | None + """The time at which the cookie is no longer valid.""" + + max_age: int | None + """The number of seconds from when the cookie was set at which it is + no longer valid. + """ + + domain: str + """The domain that the cookie was set for, or the request domain if not set.""" + + origin_only: bool + """Whether the cookie will be sent for exact domain matches only. This is ``True`` + if the ``Domain`` parameter was not present. + """ + + path: str + """The path that the cookie was set for.""" + + secure: bool | None + """The ``Secure`` parameter.""" + + http_only: bool | None + """The ``HttpOnly`` parameter.""" + + same_site: str | None + """The ``SameSite`` parameter.""" + + def _matches_request(self, server_name: str, path: str) -> bool: + return ( + server_name == self.domain + or ( + not self.origin_only + and server_name.endswith(self.domain) + and server_name[: -len(self.domain)].endswith(".") + ) + ) and ( + path == self.path + or ( + path.startswith(self.path) + and path[len(self.path) - self.path.endswith("/") :].startswith("/") + ) + ) + + def _to_request_header(self) -> str: + return f"{self.key}={self.value}" + + @classmethod + def _from_response_header(cls, server_name: str, path: str, header: str) -> te.Self: + header, _, parameters_str = header.partition(";") + key, _, value = header.partition("=") + decoded_key, decoded_value = next(parse_cookie(header).items()) # type: ignore[call-overload] + params = {} + + for item in parameters_str.split(";"): + k, sep, v = item.partition("=") + params[k.strip().lower()] = v.strip() if sep else None + + return cls( + key=key.strip(), + value=value.strip(), + decoded_key=decoded_key, + decoded_value=decoded_value, + expires=parse_date(params.get("expires")), + max_age=int(params["max-age"] or 0) if "max-age" in params else None, + domain=params.get("domain") or server_name, + origin_only="domain" not in params, + path=params.get("path") or path.rpartition("/")[0] or "/", + secure="secure" in params, + http_only="httponly" in params, + same_site=params.get("samesite"), + ) + + @property + def _storage_key(self) -> tuple[str, str, str]: + return self.domain, self.path, self.decoded_key + + @property + def _should_delete(self) -> bool: + return self.max_age == 0 or ( + self.expires is not None and self.expires.timestamp() == 0 + ) diff --git a/lib/python3.12/site-packages/werkzeug/testapp.py b/lib/python3.12/site-packages/werkzeug/testapp.py new file mode 100644 index 0000000000000000000000000000000000000000..cdf7fac1acb8266c011d842aec97d0ff2156d200 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/testapp.py @@ -0,0 +1,194 @@ +"""A small application that can be used to test a WSGI server and check +it for WSGI compliance. +""" + +from __future__ import annotations + +import importlib.metadata +import os +import sys +import typing as t +from textwrap import wrap + +from markupsafe import escape + +from .wrappers.request import Request +from .wrappers.response import Response + +TEMPLATE = """\ + + +WSGI Information + +
    +

    WSGI Information

    +

    + This page displays all available information about the WSGI server and + the underlying Python interpreter. +

    Python Interpreter

    + + + + + + +
    Python Version + %(python_version)s +
    Platform + %(platform)s [%(os)s] +
    API Version + %(api_version)s +
    Byteorder + %(byteorder)s +
    Werkzeug Version + %(werkzeug_version)s +
    +

    WSGI Environment

    + %(wsgi_env)s
    +

    Installed Eggs

    +

    + The following python packages were installed on the system as + Python eggs: +

      %(python_eggs)s
    +

    System Path

    +

    + The following paths are the current contents of the load path. The + following entries are looked up for Python packages. Note that not + all items in this path are folders. Gray and underlined items are + entries pointing to invalid resources or used by custom import hooks + such as the zip importer. +

    + Items with a bright background were expanded for display from a relative + path. If you encounter such paths in the output you might want to check + your setup as relative paths are usually problematic in multithreaded + environments. +

      %(sys_path)s
    +
    +""" + + +def iter_sys_path() -> t.Iterator[tuple[str, bool, bool]]: + if os.name == "posix": + + def strip(x: str) -> str: + prefix = os.path.expanduser("~") + if x.startswith(prefix): + x = f"~{x[len(prefix) :]}" + return x + + else: + + def strip(x: str) -> str: + return x + + cwd = os.path.abspath(os.getcwd()) + for item in sys.path: + path = os.path.join(cwd, item or os.path.curdir) + yield strip(os.path.normpath(path)), not os.path.isdir(path), path != item + + +@Request.application +def test_app(req: Request) -> Response: + """Simple test application that dumps the environment. You can use + it to check if Werkzeug is working properly: + + .. sourcecode:: pycon + + >>> from werkzeug.serving import run_simple + >>> from werkzeug.testapp import test_app + >>> run_simple('localhost', 3000, test_app) + * Running on http://localhost:3000/ + + The application displays important information from the WSGI environment, + the Python interpreter and the installed libraries. + """ + try: + import pkg_resources + except ImportError: + eggs: t.Iterable[t.Any] = () + else: + eggs = sorted( + pkg_resources.working_set, + key=lambda x: x.project_name.lower(), + ) + python_eggs = [] + for egg in eggs: + try: + version = egg.version + except (ValueError, AttributeError): + version = "unknown" + python_eggs.append( + f"
  • {escape(egg.project_name)} [{escape(version)}]" + ) + + wsgi_env = [] + sorted_environ = sorted(req.environ.items(), key=lambda x: repr(x[0]).lower()) + for key, value in sorted_environ: + value = "".join(wrap(str(escape(repr(value))))) + wsgi_env.append(f"{escape(key)}{value}") + + sys_path = [] + for item, virtual, expanded in iter_sys_path(): + css = [] + if virtual: + css.append("virtual") + if expanded: + css.append("exp") + class_str = f' class="{" ".join(css)}"' if css else "" + sys_path.append(f"{escape(item)}") + + context = { + "python_version": "
    ".join(escape(sys.version).splitlines()), + "platform": escape(sys.platform), + "os": escape(os.name), + "api_version": sys.api_version, + "byteorder": sys.byteorder, + "werkzeug_version": _get_werkzeug_version(), + "python_eggs": "\n".join(python_eggs), + "wsgi_env": "\n".join(wsgi_env), + "sys_path": "\n".join(sys_path), + } + return Response(TEMPLATE % context, mimetype="text/html") + + +_werkzeug_version = "" + + +def _get_werkzeug_version() -> str: + global _werkzeug_version + + if not _werkzeug_version: + _werkzeug_version = importlib.metadata.version("werkzeug") + + return _werkzeug_version + + +if __name__ == "__main__": + from .serving import run_simple + + run_simple("localhost", 5000, test_app, use_reloader=True) diff --git a/lib/python3.12/site-packages/werkzeug/urls.py b/lib/python3.12/site-packages/werkzeug/urls.py new file mode 100644 index 0000000000000000000000000000000000000000..5bffe392801e287f086b0505786d60740e75fd03 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/urls.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import codecs +import re +import typing as t +import urllib.parse +from urllib.parse import quote +from urllib.parse import unquote +from urllib.parse import urlencode +from urllib.parse import urlsplit +from urllib.parse import urlunsplit + +from .datastructures import iter_multi_items + + +def _codec_error_url_quote(e: UnicodeError) -> tuple[str, int]: + """Used in :func:`uri_to_iri` after unquoting to re-quote any + invalid bytes. + """ + # the docs state that UnicodeError does have these attributes, + # but mypy isn't picking them up + out = quote(e.object[e.start : e.end], safe="") # type: ignore + return out, e.end # type: ignore + + +codecs.register_error("werkzeug.url_quote", _codec_error_url_quote) + + +def _make_unquote_part(name: str, chars: str) -> t.Callable[[str], str]: + """Create a function that unquotes all percent encoded characters except those + given. This allows working with unquoted characters if possible while not changing + the meaning of a given part of a URL. + """ + choices = "|".join(f"{ord(c):02X}" for c in sorted(chars)) + pattern = re.compile(f"((?:%(?:{choices}))+)", re.I) + + def _unquote_partial(value: str) -> str: + parts = iter(pattern.split(value)) + out = [] + + for part in parts: + out.append(unquote(part, "utf-8", "werkzeug.url_quote")) + out.append(next(parts, "")) + + return "".join(out) + + _unquote_partial.__name__ = f"_unquote_{name}" + return _unquote_partial + + +# characters that should remain quoted in URL parts +# based on https://url.spec.whatwg.org/#percent-encoded-bytes +# always keep all controls, space, and % quoted +_always_unsafe = bytes((*range(0x21), 0x25, 0x7F)).decode() +_unquote_fragment = _make_unquote_part("fragment", _always_unsafe) +_unquote_query = _make_unquote_part("query", _always_unsafe + "&=+#") +_unquote_path = _make_unquote_part("path", _always_unsafe + "/?#") +_unquote_user = _make_unquote_part("user", _always_unsafe + ":@/?#") + + +def uri_to_iri(uri: str) -> str: + """Convert a URI to an IRI. All valid UTF-8 characters are unquoted, + leaving all reserved and invalid characters quoted. If the URL has + a domain, it is decoded from Punycode. + + >>> uri_to_iri("http://xn--n3h.net/p%C3%A5th?q=%C3%A8ry%DF") + 'http://\\u2603.net/p\\xe5th?q=\\xe8ry%DF' + + :param uri: The URI to convert. + + .. versionchanged:: 3.0 + Passing a tuple or bytes, and the ``charset`` and ``errors`` parameters, + are removed. + + .. versionchanged:: 2.3 + Which characters remain quoted is specific to each part of the URL. + + .. versionchanged:: 0.15 + All reserved and invalid characters remain quoted. Previously, + only some reserved characters were preserved, and invalid bytes + were replaced instead of left quoted. + + .. versionadded:: 0.6 + """ + parts = urlsplit(uri) + path = _unquote_path(parts.path) + query = _unquote_query(parts.query) + fragment = _unquote_fragment(parts.fragment) + + if parts.hostname: + netloc = _decode_idna(parts.hostname) + else: + netloc = "" + + if ":" in netloc: + netloc = f"[{netloc}]" + + if parts.port: + netloc = f"{netloc}:{parts.port}" + + if parts.username: + auth = _unquote_user(parts.username) + + if parts.password: + password = _unquote_user(parts.password) + auth = f"{auth}:{password}" + + netloc = f"{auth}@{netloc}" + + return urlunsplit((parts.scheme, netloc, path, query, fragment)) + + +def iri_to_uri(iri: str) -> str: + """Convert an IRI to a URI. All non-ASCII and unsafe characters are + quoted. If the URL has a domain, it is encoded to Punycode. + + >>> iri_to_uri('http://\\u2603.net/p\\xe5th?q=\\xe8ry%DF') + 'http://xn--n3h.net/p%C3%A5th?q=%C3%A8ry%DF' + + :param iri: The IRI to convert. + + .. versionchanged:: 3.0 + Passing a tuple or bytes, the ``charset`` and ``errors`` parameters, + and the ``safe_conversion`` parameter, are removed. + + .. versionchanged:: 2.3 + Which characters remain unquoted is specific to each part of the URL. + + .. versionchanged:: 0.15 + All reserved characters remain unquoted. Previously, only some reserved + characters were left unquoted. + + .. versionchanged:: 0.9.6 + The ``safe_conversion`` parameter was added. + + .. versionadded:: 0.6 + """ + parts = urlsplit(iri) + # safe = https://url.spec.whatwg.org/#url-path-segment-string + # as well as percent for things that are already quoted + path = quote(parts.path, safe="%!$&'()*+,/:;=@") + query = quote(parts.query, safe="%!$&'()*+,/:;=?@") + fragment = quote(parts.fragment, safe="%!#$&'()*+,/:;=?@") + + if parts.hostname: + netloc = parts.hostname.encode("idna").decode("ascii") + else: + netloc = "" + + if ":" in netloc: + netloc = f"[{netloc}]" + + if parts.port: + netloc = f"{netloc}:{parts.port}" + + if parts.username: + auth = quote(parts.username, safe="%!$&'()*+,;=") + + if parts.password: + password = quote(parts.password, safe="%!$&'()*+,;=") + auth = f"{auth}:{password}" + + netloc = f"{auth}@{netloc}" + + return urlunsplit((parts.scheme, netloc, path, query, fragment)) + + +# Python < 3.12 +# itms-services was worked around in previous iri_to_uri implementations, but +# we can tell Python directly that it needs to preserve the //. +if "itms-services" not in urllib.parse.uses_netloc: + urllib.parse.uses_netloc.append("itms-services") + + +def _decode_idna(domain: str) -> str: + try: + data = domain.encode("ascii") + except UnicodeEncodeError: + # If the domain is not ASCII, it's decoded already. + return domain + + try: + # Try decoding in one shot. + return data.decode("idna") + except UnicodeDecodeError: + pass + + # Decode each part separately, leaving invalid parts as punycode. + parts = [] + + for part in data.split(b"."): + try: + parts.append(part.decode("idna")) + except UnicodeDecodeError: + parts.append(part.decode("ascii")) + + return ".".join(parts) + + +def _urlencode(query: t.Mapping[str, str] | t.Iterable[tuple[str, str]]) -> str: + items = [x for x in iter_multi_items(query) if x[1] is not None] + # safe = https://url.spec.whatwg.org/#percent-encoded-bytes + return urlencode(items, safe="!$'()*,/:;?@") diff --git a/lib/python3.12/site-packages/werkzeug/user_agent.py b/lib/python3.12/site-packages/werkzeug/user_agent.py new file mode 100644 index 0000000000000000000000000000000000000000..17e5d3fdbf0560318981210c7c824fee10853d13 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/user_agent.py @@ -0,0 +1,47 @@ +from __future__ import annotations + + +class UserAgent: + """Represents a parsed user agent header value. + + The default implementation does no parsing, only the :attr:`string` + attribute is set. A subclass may parse the string to set the + common attributes or expose other information. Set + :attr:`werkzeug.wrappers.Request.user_agent_class` to use a + subclass. + + :param string: The header value to parse. + + .. versionadded:: 2.0 + This replaces the previous ``useragents`` module, but does not + provide a built-in parser. + """ + + platform: str | None = None + """The OS name, if it could be parsed from the string.""" + + browser: str | None = None + """The browser name, if it could be parsed from the string.""" + + version: str | None = None + """The browser version, if it could be parsed from the string.""" + + language: str | None = None + """The browser language, if it could be parsed from the string.""" + + def __init__(self, string: str) -> None: + self.string: str = string + """The original header value.""" + + def __repr__(self) -> str: + return f"<{type(self).__name__} {self.browser}/{self.version}>" + + def __str__(self) -> str: + return self.string + + def __bool__(self) -> bool: + return bool(self.browser) + + def to_header(self) -> str: + """Convert to a header value.""" + return self.string diff --git a/lib/python3.12/site-packages/werkzeug/utils.py b/lib/python3.12/site-packages/werkzeug/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..0d4a700f0859e5224ed539ee1e3f951170619d51 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/utils.py @@ -0,0 +1,684 @@ +from __future__ import annotations + +import io +import mimetypes +import os +import pkgutil +import re +import sys +import typing as t +import unicodedata +from datetime import datetime +from time import time +from urllib.parse import quote +from zlib import adler32 + +from markupsafe import escape + +from ._internal import _DictAccessorProperty +from ._internal import _missing +from ._internal import _TAccessorValue +from .datastructures import Headers +from .exceptions import NotFound +from .exceptions import RequestedRangeNotSatisfiable +from .security import _windows_device_files +from .security import safe_join +from .wsgi import wrap_file + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIEnvironment + + from .wrappers.request import Request + from .wrappers.response import Response + +_T = t.TypeVar("_T") + +_entity_re = re.compile(r"&([^;]+);") +_filename_ascii_strip_re = re.compile(r"[^A-Za-z0-9_.-]") + + +class cached_property(property, t.Generic[_T]): + """A :func:`property` that is only evaluated once. Subsequent access + returns the cached value. Setting the property sets the cached + value. Deleting the property clears the cached value, accessing it + again will evaluate it again. + + .. code-block:: python + + class Example: + @cached_property + def value(self): + # calculate something important here + return 42 + + e = Example() + e.value # evaluates + e.value # uses cache + e.value = 16 # sets cache + del e.value # clears cache + + If the class defines ``__slots__``, it must add ``_cache_{name}`` as + a slot. Alternatively, it can add ``__dict__``, but that's usually + not desirable. + + .. versionchanged:: 2.1 + Works with ``__slots__``. + + .. versionchanged:: 2.0 + ``del obj.name`` clears the cached value. + """ + + def __init__( + self, + fget: t.Callable[[t.Any], _T], + name: str | None = None, + doc: str | None = None, + ) -> None: + super().__init__(fget, doc=doc) + self.__name__ = name or fget.__name__ + self.slot_name = f"_cache_{self.__name__}" + self.__module__ = fget.__module__ + + def __set__(self, obj: object, value: _T) -> None: + if hasattr(obj, "__dict__"): + obj.__dict__[self.__name__] = value + else: + setattr(obj, self.slot_name, value) + + def __get__(self, obj: object, type: type = None) -> _T: # type: ignore + if obj is None: + return self # type: ignore + + obj_dict = getattr(obj, "__dict__", None) + + if obj_dict is not None: + value: _T = obj_dict.get(self.__name__, _missing) + else: + value = getattr(obj, self.slot_name, _missing) # type: ignore[arg-type] + + if value is _missing: + value = self.fget(obj) # type: ignore + + if obj_dict is not None: + obj.__dict__[self.__name__] = value + else: + setattr(obj, self.slot_name, value) + + return value + + def __delete__(self, obj: object) -> None: + if hasattr(obj, "__dict__"): + del obj.__dict__[self.__name__] + else: + setattr(obj, self.slot_name, _missing) + + +class environ_property(_DictAccessorProperty[_TAccessorValue]): + """Maps request attributes to environment variables. This works not only + for the Werkzeug request object, but also any other class with an + environ attribute: + + >>> class Test(object): + ... environ = {'key': 'value'} + ... test = environ_property('key') + >>> var = Test() + >>> var.test + 'value' + + If you pass it a second value it's used as default if the key does not + exist, the third one can be a converter that takes a value and converts + it. If it raises :exc:`ValueError` or :exc:`TypeError` the default value + is used. If no default value is provided `None` is used. + + Per default the property is read only. You have to explicitly enable it + by passing ``read_only=False`` to the constructor. + """ + + read_only = True + + def lookup(self, obj: Request) -> WSGIEnvironment: + return obj.environ + + +class header_property(_DictAccessorProperty[_TAccessorValue]): + """Like `environ_property` but for headers.""" + + def lookup(self, obj: Request | Response) -> Headers: # type: ignore[override] + return obj.headers + + +# https://cgit.freedesktop.org/xdg/shared-mime-info/tree/freedesktop.org.xml.in +# https://www.iana.org/assignments/media-types/media-types.xhtml +# Types listed in the XDG mime info that have a charset in the IANA registration. +_charset_mimetypes = { + "application/ecmascript", + "application/javascript", + "application/sql", + "application/xml", + "application/xml-dtd", + "application/xml-external-parsed-entity", +} + + +def get_content_type(mimetype: str, charset: str) -> str: + """Returns the full content type string with charset for a mimetype. + + If the mimetype represents text, the charset parameter will be + appended, otherwise the mimetype is returned unchanged. + + :param mimetype: The mimetype to be used as content type. + :param charset: The charset to be appended for text mimetypes. + :return: The content type. + + .. versionchanged:: 0.15 + Any type that ends with ``+xml`` gets a charset, not just those + that start with ``application/``. Known text types such as + ``application/javascript`` are also given charsets. + """ + if ( + mimetype.startswith("text/") + or mimetype in _charset_mimetypes + or mimetype.endswith("+xml") + ): + mimetype += f"; charset={charset}" + + return mimetype + + +def secure_filename(filename: str) -> str: + r"""Pass it a filename and it will return a secure version of it. This + filename can then safely be stored on a regular file system and passed + to :func:`os.path.join`. The filename returned is an ASCII only string + for maximum portability. + + On windows systems the function also makes sure that the file is not + named after one of the special device files. + + >>> secure_filename("My cool movie.mov") + 'My_cool_movie.mov' + >>> secure_filename("../../../etc/passwd") + 'etc_passwd' + >>> secure_filename('i contain cool \xfcml\xe4uts.txt') + 'i_contain_cool_umlauts.txt' + + The function might return an empty filename. It's your responsibility + to ensure that the filename is unique and that you abort or + generate a random filename if the function returned an empty one. + + .. versionadded:: 0.5 + + :param filename: the filename to secure + """ + filename = unicodedata.normalize("NFKD", filename) + filename = filename.encode("ascii", "ignore").decode("ascii") + + for sep in os.sep, os.path.altsep: + if sep: + filename = filename.replace(sep, " ") + filename = str(_filename_ascii_strip_re.sub("", "_".join(filename.split()))).strip( + "._" + ) + + # on nt a couple of special files are present in each folder. We + # have to ensure that the target file is not such a filename. In + # this case we prepend an underline + if ( + os.name == "nt" + and filename + and filename.split(".")[0].upper() in _windows_device_files + ): + filename = f"_{filename}" + + return filename + + +def redirect( + location: str, code: int = 302, Response: type[Response] | None = None +) -> Response: + """Returns a response object (a WSGI application) that, if called, + redirects the client to the target location. Supported codes are + 301, 302, 303, 305, 307, and 308. 300 is not supported because + it's not a real redirect and 304 because it's the answer for a + request with a request with defined If-Modified-Since headers. + + .. versionadded:: 0.6 + The location can now be a unicode string that is encoded using + the :func:`iri_to_uri` function. + + .. versionadded:: 0.10 + The class used for the Response object can now be passed in. + + :param location: the location the response should redirect to. + :param code: the redirect status code. defaults to 302. + :param class Response: a Response class to use when instantiating a + response. The default is :class:`werkzeug.wrappers.Response` if + unspecified. + """ + if Response is None: + from .wrappers import Response + + html_location = escape(location) + response = Response( # type: ignore[misc] + "\n" + "\n" + "Redirecting...\n" + "

    Redirecting...

    \n" + "

    You should be redirected automatically to the target URL: " + f'{html_location}. If not, click the link.\n', + code, + mimetype="text/html", + ) + response.headers["Location"] = location + return response + + +def append_slash_redirect(environ: WSGIEnvironment, code: int = 308) -> Response: + """Redirect to the current URL with a slash appended. + + If the current URL is ``/user/42``, the redirect URL will be + ``42/``. When joined to the current URL during response + processing or by the browser, this will produce ``/user/42/``. + + The behavior is undefined if the path ends with a slash already. If + called unconditionally on a URL, it may produce a redirect loop. + + :param environ: Use the path and query from this WSGI environment + to produce the redirect URL. + :param code: the status code for the redirect. + + .. versionchanged:: 2.1 + Produce a relative URL that only modifies the last segment. + Relevant when the current path has multiple segments. + + .. versionchanged:: 2.1 + The default status code is 308 instead of 301. This preserves + the request method and body. + """ + tail = environ["PATH_INFO"].rpartition("/")[2] + + if not tail: + new_path = "./" + else: + new_path = f"{tail}/" + + query_string = environ.get("QUERY_STRING") + + if query_string: + new_path = f"{new_path}?{query_string}" + + return redirect(new_path, code) + + +def send_file( + path_or_file: os.PathLike[str] | str | t.IO[bytes], + environ: WSGIEnvironment, + mimetype: str | None = None, + as_attachment: bool = False, + download_name: str | None = None, + conditional: bool = True, + etag: bool | str = True, + last_modified: datetime | int | float | None = None, + max_age: None | (int | t.Callable[[str | None], int | None]) = None, + use_x_sendfile: bool = False, + response_class: type[Response] | None = None, + _root_path: os.PathLike[str] | str | None = None, +) -> Response: + """Send the contents of a file to the client. + + The first argument can be a file path or a file-like object. Paths + are preferred in most cases because Werkzeug can manage the file and + get extra information from the path. Passing a file-like object + requires that the file is opened in binary mode, and is mostly + useful when building a file in memory with :class:`io.BytesIO`. + + Never pass file paths provided by a user. The path is assumed to be + trusted, so a user could craft a path to access a file you didn't + intend. Use :func:`send_from_directory` to safely serve user-provided paths. + + If the WSGI server sets a ``file_wrapper`` in ``environ``, it is + used, otherwise Werkzeug's built-in wrapper is used. Alternatively, + if the HTTP server supports ``X-Sendfile``, ``use_x_sendfile=True`` + will tell the server to send the given path, which is much more + efficient than reading it in Python. + + :param path_or_file: The path to the file to send, relative to the + current working directory if a relative path is given. + Alternatively, a file-like object opened in binary mode. Make + sure the file pointer is seeked to the start of the data. + :param environ: The WSGI environ for the current request. + :param mimetype: The MIME type to send for the file. If not + provided, it will try to detect it from the file name. + :param as_attachment: Indicate to a browser that it should offer to + save the file instead of displaying it. + :param download_name: The default name browsers will use when saving + the file. Defaults to the passed file name. + :param conditional: Enable conditional and range responses based on + request headers. Requires passing a file path and ``environ``. + :param etag: Calculate an ETag for the file, which requires passing + a file path. Can also be a string to use instead. + :param last_modified: The last modified time to send for the file, + in seconds. If not provided, it will try to detect it from the + file path. + :param max_age: How long the client should cache the file, in + seconds. If set, ``Cache-Control`` will be ``public``, otherwise + it will be ``no-cache`` to prefer conditional caching. + :param use_x_sendfile: Set the ``X-Sendfile`` header to let the + server to efficiently send the file. Requires support from the + HTTP server. Requires passing a file path. + :param response_class: Build the response using this class. Defaults + to :class:`~werkzeug.wrappers.Response`. + :param _root_path: Do not use. For internal use only. Use + :func:`send_from_directory` to safely send files under a path. + + .. versionchanged:: 2.0.2 + ``send_file`` only sets a detected ``Content-Encoding`` if + ``as_attachment`` is disabled. + + .. versionadded:: 2.0 + Adapted from Flask's implementation. + + .. versionchanged:: 2.0 + ``download_name`` replaces Flask's ``attachment_filename`` + parameter. If ``as_attachment=False``, it is passed with + ``Content-Disposition: inline`` instead. + + .. versionchanged:: 2.0 + ``max_age`` replaces Flask's ``cache_timeout`` parameter. + ``conditional`` is enabled and ``max_age`` is not set by + default. + + .. versionchanged:: 2.0 + ``etag`` replaces Flask's ``add_etags`` parameter. It can be a + string to use instead of generating one. + + .. versionchanged:: 2.0 + If an encoding is returned when guessing ``mimetype`` from + ``download_name``, set the ``Content-Encoding`` header. + """ + if response_class is None: + from .wrappers import Response + + response_class = Response + + path: str | None = None + file: t.IO[bytes] | None = None + size: int | None = None + mtime: float | None = None + headers = Headers() + + if isinstance(path_or_file, (os.PathLike, str)) or hasattr( + path_or_file, "__fspath__" + ): + path_or_file = t.cast("t.Union[os.PathLike[str], str]", path_or_file) + + # Flask will pass app.root_path, allowing its send_file wrapper + # to not have to deal with paths. + if _root_path is not None: + path = os.path.join(_root_path, path_or_file) + else: + path = os.path.abspath(path_or_file) + + stat = os.stat(path) + size = stat.st_size + mtime = stat.st_mtime + else: + file = path_or_file + + if download_name is None and path is not None: + download_name = os.path.basename(path) + + if mimetype is None: + if download_name is None: + raise TypeError( + "Unable to detect the MIME type because a file name is" + " not available. Either set 'download_name', pass a" + " path instead of a file, or set 'mimetype'." + ) + + mimetype, encoding = mimetypes.guess_type(download_name) + + if mimetype is None: + mimetype = "application/octet-stream" + + # Don't send encoding for attachments, it causes browsers to + # save decompress tar.gz files. + if encoding is not None and not as_attachment: + headers.set("Content-Encoding", encoding) + + if download_name is not None: + try: + download_name.encode("ascii") + except UnicodeEncodeError: + simple = unicodedata.normalize("NFKD", download_name) + simple = simple.encode("ascii", "ignore").decode("ascii") + # safe = RFC 5987 attr-char + quoted = quote(download_name, safe="!#$&+-.^_`|~") + names = {"filename": simple, "filename*": f"UTF-8''{quoted}"} + else: + names = {"filename": download_name} + + value = "attachment" if as_attachment else "inline" + headers.set("Content-Disposition", value, **names) + elif as_attachment: + raise TypeError( + "No name provided for attachment. Either set" + " 'download_name' or pass a path instead of a file." + ) + + if use_x_sendfile and path is not None: + headers["X-Sendfile"] = path + data = None + else: + if file is None: + file = open(path, "rb") # type: ignore + elif isinstance(file, io.BytesIO): + size = file.getbuffer().nbytes + elif isinstance(file, io.TextIOBase): + raise ValueError("Files must be opened in binary mode or use BytesIO.") + + data = wrap_file(environ, file) + + rv = response_class( + data, mimetype=mimetype, headers=headers, direct_passthrough=True + ) + + if size is not None: + rv.content_length = size + + if last_modified is not None: + rv.last_modified = last_modified # type: ignore + elif mtime is not None: + rv.last_modified = mtime # type: ignore + + rv.cache_control.no_cache = True + + # Flask will pass app.get_send_file_max_age, allowing its send_file + # wrapper to not have to deal with paths. + if callable(max_age): + max_age = max_age(path) + + if max_age is not None: + if max_age > 0: + rv.cache_control.no_cache = None + rv.cache_control.public = True + + rv.cache_control.max_age = max_age + rv.expires = int(time() + max_age) # type: ignore + + if isinstance(etag, str): + rv.set_etag(etag) + elif etag and path is not None: + check = adler32(path.encode()) & 0xFFFFFFFF + rv.set_etag(f"{mtime}-{size}-{check}") + + if conditional: + try: + rv = rv.make_conditional(environ, accept_ranges=True, complete_length=size) + except RequestedRangeNotSatisfiable: + if file is not None: + file.close() + + raise + + # Some x-sendfile implementations incorrectly ignore the 304 + # status code and send the file anyway. + if rv.status_code == 304: + rv.headers.pop("x-sendfile", None) + + return rv + + +def send_from_directory( + directory: os.PathLike[str] | str, + path: os.PathLike[str] | str, + environ: WSGIEnvironment, + **kwargs: t.Any, +) -> Response: + """Send a file from within a directory using :func:`send_file`. + + This is a secure way to serve files from a folder, such as static + files or uploads. Uses :func:`~werkzeug.security.safe_join` to + ensure the path coming from the client is not maliciously crafted to + point outside the specified directory. + + If the final path does not point to an existing regular file, + returns a 404 :exc:`~werkzeug.exceptions.NotFound` error. + + :param directory: The directory that ``path`` must be located under. This *must not* + be a value provided by the client, otherwise it becomes insecure. + :param path: The path to the file to send, relative to ``directory``. This is the + part of the path provided by the client, which is checked for security. + :param environ: The WSGI environ for the current request. + :param kwargs: Arguments to pass to :func:`send_file`. + + .. versionadded:: 2.0 + Adapted from Flask's implementation. + """ + path_str = safe_join(os.fspath(directory), os.fspath(path)) + + if path_str is None: + raise NotFound() + + # Flask will pass app.root_path, allowing its send_from_directory + # wrapper to not have to deal with paths. + if "_root_path" in kwargs: + path_str = os.path.join(kwargs["_root_path"], path_str) + + if not os.path.isfile(path_str): + raise NotFound() + + return send_file(path_str, environ, **kwargs) + + +def import_string(import_name: str, silent: bool = False) -> t.Any: + """Imports an object based on a string. This is useful if you want to + use import paths as endpoints or something similar. An import path can + be specified either in dotted notation (``xml.sax.saxutils.escape``) + or with a colon as object delimiter (``xml.sax.saxutils:escape``). + + If `silent` is True the return value will be `None` if the import fails. + + :param import_name: the dotted name for the object to import. + :param silent: if set to `True` import errors are ignored and + `None` is returned instead. + :return: imported object + """ + import_name = import_name.replace(":", ".") + try: + try: + __import__(import_name) + except ImportError: + if "." not in import_name: + raise + else: + return sys.modules[import_name] + + module_name, obj_name = import_name.rsplit(".", 1) + module = __import__(module_name, globals(), locals(), [obj_name]) + try: + return getattr(module, obj_name) + except AttributeError as e: + raise ImportError(e) from None + + except ImportError as e: + if not silent: + raise ImportStringError(import_name, e).with_traceback( + sys.exc_info()[2] + ) from None + + return None + + +def find_modules( + import_path: str, include_packages: bool = False, recursive: bool = False +) -> t.Iterator[str]: + """Finds all the modules below a package. This can be useful to + automatically import all views / controllers so that their metaclasses / + function decorators have a chance to register themselves on the + application. + + Packages are not returned unless `include_packages` is `True`. This can + also recursively list modules but in that case it will import all the + packages to get the correct load path of that module. + + :param import_path: the dotted name for the package to find child modules. + :param include_packages: set to `True` if packages should be returned, too. + :param recursive: set to `True` if recursion should happen. + :return: generator + """ + module = import_string(import_path) + path = getattr(module, "__path__", None) + if path is None: + raise ValueError(f"{import_path!r} is not a package") + basename = f"{module.__name__}." + for _importer, modname, ispkg in pkgutil.iter_modules(path): + modname = basename + modname + if ispkg: + if include_packages: + yield modname + if recursive: + yield from find_modules(modname, include_packages, True) + else: + yield modname + + +class ImportStringError(ImportError): + """Provides information about a failed :func:`import_string` attempt.""" + + #: String in dotted notation that failed to be imported. + import_name: str + #: Wrapped exception. + exception: BaseException + + def __init__(self, import_name: str, exception: BaseException) -> None: + self.import_name = import_name + self.exception = exception + msg = import_name + name = "" + tracked = [] + for part in import_name.replace(":", ".").split("."): + name = f"{name}.{part}" if name else part + imported = import_string(name, silent=True) + if imported: + tracked.append((name, getattr(imported, "__file__", None))) + else: + track = [f"- {n!r} found in {i!r}." for n, i in tracked] + track.append(f"- {name!r} not found.") + track_str = "\n".join(track) + msg = ( + f"import_string() failed for {import_name!r}. Possible reasons" + f" are:\n\n" + "- missing __init__.py in a package;\n" + "- package or module path not included in sys.path;\n" + "- duplicated package or module name taking precedence in" + " sys.path;\n" + "- missing module, class, function or variable;\n\n" + f"Debugged import:\n\n{track_str}\n\n" + f"Original exception:\n\n{type(exception).__name__}: {exception}" + ) + break + + super().__init__(msg) + + def __repr__(self) -> str: + return f"<{type(self).__name__}({self.import_name!r}, {self.exception!r})>" diff --git a/lib/python3.12/site-packages/werkzeug/wrappers/__init__.py b/lib/python3.12/site-packages/werkzeug/wrappers/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..b36f228f2cef05a53bff1e07f241a6bd60576fac --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/wrappers/__init__.py @@ -0,0 +1,3 @@ +from .request import Request as Request +from .response import Response as Response +from .response import ResponseStream as ResponseStream diff --git a/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8b9d022f6e8058ef69fdd06f322c6ad09ce84df Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/request.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/request.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fd2c97b6f53d0ae8274b3c13b87ec081f17aa428 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/request.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/response.cpython-312.pyc b/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/response.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1f5835e4fb3603b924566d66dd965d8e01489113 Binary files /dev/null and b/lib/python3.12/site-packages/werkzeug/wrappers/__pycache__/response.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/werkzeug/wrappers/request.py b/lib/python3.12/site-packages/werkzeug/wrappers/request.py new file mode 100644 index 0000000000000000000000000000000000000000..9f1eee118421ed35ded9646573eff88c16227bf2 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/wrappers/request.py @@ -0,0 +1,650 @@ +from __future__ import annotations + +import collections.abc as cabc +import functools +import json +import typing as t +from io import BytesIO + +from .._internal import _wsgi_decoding_dance +from ..datastructures import CombinedMultiDict +from ..datastructures import EnvironHeaders +from ..datastructures import FileStorage +from ..datastructures import ImmutableMultiDict +from ..datastructures import iter_multi_items +from ..datastructures import MultiDict +from ..exceptions import BadRequest +from ..exceptions import UnsupportedMediaType +from ..formparser import default_stream_factory +from ..formparser import FormDataParser +from ..sansio.request import Request as _SansIORequest +from ..utils import cached_property +from ..utils import environ_property +from ..wsgi import _get_server +from ..wsgi import get_input_stream + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +class Request(_SansIORequest): + """Represents an incoming WSGI HTTP request, with headers and body + taken from the WSGI environment. Has properties and methods for + using the functionality defined by various HTTP specs. The data in + requests object is read-only. + + Text data is assumed to use UTF-8 encoding, which should be true for + the vast majority of modern clients. Using an encoding set by the + client is unsafe in Python due to extra encodings it provides, such + as ``zip``. To change the assumed encoding, subclass and replace + :attr:`charset`. + + :param environ: The WSGI environ is generated by the WSGI server and + contains information about the server configuration and client + request. + :param populate_request: Add this request object to the WSGI environ + as ``environ['werkzeug.request']``. Can be useful when + debugging. + :param shallow: Makes reading from :attr:`stream` (and any method + that would read from it) raise a :exc:`RuntimeError`. Useful to + prevent consuming the form data in middleware, which would make + it unavailable to the final application. + + .. versionchanged:: 3.0 + The ``charset``, ``url_charset``, and ``encoding_errors`` parameters + were removed. + + .. versionchanged:: 2.1 + Old ``BaseRequest`` and mixin classes were removed. + + .. versionchanged:: 2.1 + Remove the ``disable_data_descriptor`` attribute. + + .. versionchanged:: 2.0 + Combine ``BaseRequest`` and mixins into a single ``Request`` + class. + + .. versionchanged:: 0.5 + Read-only mode is enforced with immutable classes for all data. + """ + + #: the maximum content length. This is forwarded to the form data + #: parsing function (:func:`parse_form_data`). When set and the + #: :attr:`form` or :attr:`files` attribute is accessed and the + #: parsing fails because more than the specified value is transmitted + #: a :exc:`~werkzeug.exceptions.RequestEntityTooLarge` exception is raised. + #: + #: .. versionadded:: 0.5 + max_content_length: int | None = None + + #: the maximum form field size. This is forwarded to the form data + #: parsing function (:func:`parse_form_data`). When set and the + #: :attr:`form` or :attr:`files` attribute is accessed and the + #: data in memory for post data is longer than the specified value a + #: :exc:`~werkzeug.exceptions.RequestEntityTooLarge` exception is raised. + #: + #: .. versionchanged:: 3.1 + #: Defaults to 500kB instead of unlimited. + #: + #: .. versionadded:: 0.5 + max_form_memory_size: int | None = 500_000 + + #: The maximum number of multipart parts to parse, passed to + #: :attr:`form_data_parser_class`. Parsing form data with more than this + #: many parts will raise :exc:`~.RequestEntityTooLarge`. + #: + #: .. versionadded:: 2.2.3 + max_form_parts = 1000 + + #: The form data parser that should be used. Can be replaced to customize + #: the form date parsing. + form_data_parser_class: type[FormDataParser] = FormDataParser + + #: The WSGI environment containing HTTP headers and information from + #: the WSGI server. + environ: WSGIEnvironment + + #: Set when creating the request object. If ``True``, reading from + #: the request body will cause a ``RuntimeException``. Useful to + #: prevent modifying the stream from middleware. + shallow: bool + + def __init__( + self, + environ: WSGIEnvironment, + populate_request: bool = True, + shallow: bool = False, + ) -> None: + super().__init__( + method=environ.get("REQUEST_METHOD", "GET"), + scheme=environ.get("wsgi.url_scheme", "http"), + server=_get_server(environ), + root_path=_wsgi_decoding_dance(environ.get("SCRIPT_NAME") or ""), + path=_wsgi_decoding_dance(environ.get("PATH_INFO") or ""), + query_string=environ.get("QUERY_STRING", "").encode("latin1"), + headers=EnvironHeaders(environ), + remote_addr=environ.get("REMOTE_ADDR"), + ) + self.environ = environ + self.shallow = shallow + + if populate_request and not shallow: + self.environ["werkzeug.request"] = self + + @classmethod + def from_values(cls, *args: t.Any, **kwargs: t.Any) -> Request: + """Create a new request object based on the values provided. If + environ is given missing values are filled from there. This method is + useful for small scripts when you need to simulate a request from an URL. + Do not use this method for unittesting, there is a full featured client + object (:class:`Client`) that allows to create multipart requests, + support for cookies etc. + + This accepts the same options as the + :class:`~werkzeug.test.EnvironBuilder`. + + .. versionchanged:: 0.5 + This method now accepts the same arguments as + :class:`~werkzeug.test.EnvironBuilder`. Because of this the + `environ` parameter is now called `environ_overrides`. + + :return: request object + """ + from ..test import EnvironBuilder + + builder = EnvironBuilder(*args, **kwargs) + try: + return builder.get_request(cls) + finally: + builder.close() + + @classmethod + def application(cls, f: t.Callable[[Request], WSGIApplication]) -> WSGIApplication: + """Decorate a function as responder that accepts the request as + the last argument. This works like the :func:`responder` + decorator but the function is passed the request object as the + last argument and the request object will be closed + automatically:: + + @Request.application + def my_wsgi_app(request): + return Response('Hello World!') + + As of Werkzeug 0.14 HTTP exceptions are automatically caught and + converted to responses instead of failing. + + :param f: the WSGI callable to decorate + :return: a new WSGI callable + """ + #: return a callable that wraps the -2nd argument with the request + #: and calls the function with all the arguments up to that one and + #: the request. The return value is then called with the latest + #: two arguments. This makes it possible to use this decorator for + #: both standalone WSGI functions as well as bound methods and + #: partially applied functions. + from ..exceptions import HTTPException + + @functools.wraps(f) + def application(*args: t.Any) -> cabc.Iterable[bytes]: + request = cls(args[-2]) + with request: + try: + resp = f(*args[:-2] + (request,)) + except HTTPException as e: + resp = t.cast("WSGIApplication", e.get_response(args[-2])) + return resp(*args[-2:]) + + return t.cast("WSGIApplication", application) + + def _get_file_stream( + self, + total_content_length: int | None, + content_type: str | None, + filename: str | None = None, + content_length: int | None = None, + ) -> t.IO[bytes]: + """Called to get a stream for the file upload. + + This must provide a file-like class with `read()`, `readline()` + and `seek()` methods that is both writeable and readable. + + The default implementation returns a temporary file if the total + content length is higher than 500KB. Because many browsers do not + provide a content length for the files only the total content + length matters. + + :param total_content_length: the total content length of all the + data in the request combined. This value + is guaranteed to be there. + :param content_type: the mimetype of the uploaded file. + :param filename: the filename of the uploaded file. May be `None`. + :param content_length: the length of this file. This value is usually + not provided because webbrowsers do not provide + this value. + """ + return default_stream_factory( + total_content_length=total_content_length, + filename=filename, + content_type=content_type, + content_length=content_length, + ) + + @property + def want_form_data_parsed(self) -> bool: + """``True`` if the request method carries content. By default + this is true if a ``Content-Type`` is sent. + + .. versionadded:: 0.8 + """ + return bool(self.environ.get("CONTENT_TYPE")) + + def make_form_data_parser(self) -> FormDataParser: + """Creates the form data parser. Instantiates the + :attr:`form_data_parser_class` with some parameters. + + .. versionadded:: 0.8 + """ + return self.form_data_parser_class( + stream_factory=self._get_file_stream, + max_form_memory_size=self.max_form_memory_size, + max_content_length=self.max_content_length, + max_form_parts=self.max_form_parts, + cls=self.parameter_storage_class, + ) + + def _load_form_data(self) -> None: + """Method used internally to retrieve submitted data. After calling + this sets `form` and `files` on the request object to multi dicts + filled with the incoming form data. As a matter of fact the input + stream will be empty afterwards. You can also call this method to + force the parsing of the form data. + + .. versionadded:: 0.8 + """ + # abort early if we have already consumed the stream + if "form" in self.__dict__: + return + + if self.want_form_data_parsed: + parser = self.make_form_data_parser() + data = parser.parse( + self._get_stream_for_parsing(), + self.mimetype, + self.content_length, + self.mimetype_params, + ) + else: + data = ( + self.stream, + self.parameter_storage_class(), + self.parameter_storage_class(), + ) + + # inject the values into the instance dict so that we bypass + # our cached_property non-data descriptor. + d = self.__dict__ + d["stream"], d["form"], d["files"] = data + + def _get_stream_for_parsing(self) -> t.IO[bytes]: + """This is the same as accessing :attr:`stream` with the difference + that if it finds cached data from calling :meth:`get_data` first it + will create a new stream out of the cached data. + + .. versionadded:: 0.9.3 + """ + cached_data = getattr(self, "_cached_data", None) + if cached_data is not None: + return BytesIO(cached_data) + return self.stream + + def close(self) -> None: + """Closes associated resources of this request object. This + closes all file handles explicitly. You can also use the request + object in a with statement which will automatically close it. + + .. versionadded:: 0.9 + """ + files = self.__dict__.get("files") + for _key, value in iter_multi_items(files or ()): + value.close() + + def __enter__(self) -> Request: + return self + + def __exit__(self, exc_type, exc_value, tb) -> None: # type: ignore + self.close() + + @cached_property + def stream(self) -> t.IO[bytes]: + """The WSGI input stream, with safety checks. This stream can only be consumed + once. + + Use :meth:`get_data` to get the full data as bytes or text. The :attr:`data` + attribute will contain the full bytes only if they do not represent form data. + The :attr:`form` attribute will contain the parsed form data in that case. + + Unlike :attr:`input_stream`, this stream guards against infinite streams or + reading past :attr:`content_length` or :attr:`max_content_length`. + + If ``max_content_length`` is set, it can be enforced on streams if + ``wsgi.input_terminated`` is set. Otherwise, an empty stream is returned. + + If the limit is reached before the underlying stream is exhausted (such as a + file that is too large, or an infinite stream), the remaining contents of the + stream cannot be read safely. Depending on how the server handles this, clients + may show a "connection reset" failure instead of seeing the 413 response. + + .. versionchanged:: 2.3 + Check ``max_content_length`` preemptively and while reading. + + .. versionchanged:: 0.9 + The stream is always set (but may be consumed) even if form parsing was + accessed first. + """ + if self.shallow: + raise RuntimeError( + "This request was created with 'shallow=True', reading" + " from the input stream is disabled." + ) + + return get_input_stream( + self.environ, max_content_length=self.max_content_length + ) + + input_stream = environ_property[t.IO[bytes]]( + "wsgi.input", + doc="""The raw WSGI input stream, without any safety checks. + + This is dangerous to use. It does not guard against infinite streams or reading + past :attr:`content_length` or :attr:`max_content_length`. + + Use :attr:`stream` instead. + """, + ) + + @cached_property + def data(self) -> bytes: + """The raw data read from :attr:`stream`. Will be empty if the request + represents form data. + + To get the raw data even if it represents form data, use :meth:`get_data`. + """ + return self.get_data(parse_form_data=True) + + @t.overload + def get_data( + self, + cache: bool = True, + as_text: t.Literal[False] = False, + parse_form_data: bool = False, + ) -> bytes: ... + + @t.overload + def get_data( + self, + cache: bool = True, + as_text: t.Literal[True] = ..., + parse_form_data: bool = False, + ) -> str: ... + + def get_data( + self, cache: bool = True, as_text: bool = False, parse_form_data: bool = False + ) -> bytes | str: + """This reads the buffered incoming data from the client into one + bytes object. By default this is cached but that behavior can be + changed by setting `cache` to `False`. + + Usually it's a bad idea to call this method without checking the + content length first as a client could send dozens of megabytes or more + to cause memory problems on the server. + + Note that if the form data was already parsed this method will not + return anything as form data parsing does not cache the data like + this method does. To implicitly invoke form data parsing function + set `parse_form_data` to `True`. When this is done the return value + of this method will be an empty string if the form parser handles + the data. This generally is not necessary as if the whole data is + cached (which is the default) the form parser will used the cached + data to parse the form data. Please be generally aware of checking + the content length first in any case before calling this method + to avoid exhausting server memory. + + If `as_text` is set to `True` the return value will be a decoded + string. + + .. versionadded:: 0.9 + """ + rv = getattr(self, "_cached_data", None) + if rv is None: + if parse_form_data: + self._load_form_data() + rv = self.stream.read() + if cache: + self._cached_data = rv + if as_text: + rv = rv.decode(errors="replace") + return rv + + @cached_property + def form(self) -> ImmutableMultiDict[str, str]: + """The form parameters. By default an + :class:`~werkzeug.datastructures.ImmutableMultiDict` + is returned from this function. This can be changed by setting + :attr:`parameter_storage_class` to a different type. This might + be necessary if the order of the form data is important. + + Please keep in mind that file uploads will not end up here, but instead + in the :attr:`files` attribute. + + .. versionchanged:: 0.9 + + Previous to Werkzeug 0.9 this would only contain form data for POST + and PUT requests. + """ + self._load_form_data() + return self.form + + @cached_property + def values(self) -> CombinedMultiDict[str, str]: + """A :class:`werkzeug.datastructures.CombinedMultiDict` that + combines :attr:`args` and :attr:`form`. + + For GET requests, only ``args`` are present, not ``form``. + + .. versionchanged:: 2.0 + For GET requests, only ``args`` are present, not ``form``. + """ + sources = [self.args] + + if self.method != "GET": + # GET requests can have a body, and some caching proxies + # might not treat that differently than a normal GET + # request, allowing form data to "invisibly" affect the + # cache without indication in the query string / URL. + sources.append(self.form) + + args = [] + + for d in sources: + if not isinstance(d, MultiDict): + d = MultiDict(d) + + args.append(d) + + return CombinedMultiDict(args) + + @cached_property + def files(self) -> ImmutableMultiDict[str, FileStorage]: + """:class:`~werkzeug.datastructures.MultiDict` object containing + all uploaded files. Each key in :attr:`files` is the name from the + ````. Each value in :attr:`files` is a + Werkzeug :class:`~werkzeug.datastructures.FileStorage` object. + + It basically behaves like a standard file object you know from Python, + with the difference that it also has a + :meth:`~werkzeug.datastructures.FileStorage.save` function that can + store the file on the filesystem. + + Note that :attr:`files` will only contain data if the request method was + POST, PUT or PATCH and the ``

    `` that posted to the request had + ``enctype="multipart/form-data"``. It will be empty otherwise. + + See the :class:`~werkzeug.datastructures.MultiDict` / + :class:`~werkzeug.datastructures.FileStorage` documentation for + more details about the used data structure. + """ + self._load_form_data() + return self.files + + @property + def script_root(self) -> str: + """Alias for :attr:`self.root_path`. ``environ["SCRIPT_NAME"]`` + without a trailing slash. + """ + return self.root_path + + @cached_property + def url_root(self) -> str: + """Alias for :attr:`root_url`. The URL with scheme, host, and + root path. For example, ``https://example.com/app/``. + """ + return self.root_url + + remote_user = environ_property[str]( + "REMOTE_USER", + doc="""If the server supports user authentication, and the + script is protected, this attribute contains the username the + user has authenticated as.""", + ) + is_multithread = environ_property[bool]( + "wsgi.multithread", + doc="""boolean that is `True` if the application is served by a + multithreaded WSGI server.""", + ) + is_multiprocess = environ_property[bool]( + "wsgi.multiprocess", + doc="""boolean that is `True` if the application is served by a + WSGI server that spawns multiple processes.""", + ) + is_run_once = environ_property[bool]( + "wsgi.run_once", + doc="""boolean that is `True` if the application will be + executed only once in a process lifetime. This is the case for + CGI for example, but it's not guaranteed that the execution only + happens one time.""", + ) + + # JSON + + #: A module or other object that has ``dumps`` and ``loads`` + #: functions that match the API of the built-in :mod:`json` module. + json_module = json + + @property + def json(self) -> t.Any: + """The parsed JSON data if :attr:`mimetype` indicates JSON + (:mimetype:`application/json`, see :attr:`is_json`). + + Calls :meth:`get_json` with default arguments. + + If the request content type is not ``application/json``, this + will raise a 415 Unsupported Media Type error. + + .. versionchanged:: 2.3 + Raise a 415 error instead of 400. + + .. versionchanged:: 2.1 + Raise a 400 error if the content type is incorrect. + """ + return self.get_json() + + # Cached values for ``(silent=False, silent=True)``. Initialized + # with sentinel values. + _cached_json: tuple[t.Any, t.Any] = (Ellipsis, Ellipsis) + + @t.overload + def get_json( + self, force: bool = ..., silent: t.Literal[False] = ..., cache: bool = ... + ) -> t.Any: ... + + @t.overload + def get_json( + self, force: bool = ..., silent: bool = ..., cache: bool = ... + ) -> t.Any | None: ... + + def get_json( + self, force: bool = False, silent: bool = False, cache: bool = True + ) -> t.Any | None: + """Parse :attr:`data` as JSON. + + If the mimetype does not indicate JSON + (:mimetype:`application/json`, see :attr:`is_json`), or parsing + fails, :meth:`on_json_loading_failed` is called and + its return value is used as the return value. By default this + raises a 415 Unsupported Media Type resp. + + :param force: Ignore the mimetype and always try to parse JSON. + :param silent: Silence mimetype and parsing errors, and + return ``None`` instead. + :param cache: Store the parsed JSON to return for subsequent + calls. + + .. versionchanged:: 2.3 + Raise a 415 error instead of 400. + + .. versionchanged:: 2.1 + Raise a 400 error if the content type is incorrect. + """ + if cache and self._cached_json[silent] is not Ellipsis: + return self._cached_json[silent] + + if not (force or self.is_json): + if not silent: + return self.on_json_loading_failed(None) + else: + return None + + data = self.get_data(cache=cache) + + try: + rv = self.json_module.loads(data) + except ValueError as e: + if silent: + rv = None + + if cache: + normal_rv, _ = self._cached_json + self._cached_json = (normal_rv, rv) + else: + rv = self.on_json_loading_failed(e) + + if cache: + _, silent_rv = self._cached_json + self._cached_json = (rv, silent_rv) + else: + if cache: + self._cached_json = (rv, rv) + + return rv + + def on_json_loading_failed(self, e: ValueError | None) -> t.Any: + """Called if :meth:`get_json` fails and isn't silenced. + + If this method returns a value, it is used as the return value + for :meth:`get_json`. The default implementation raises + :exc:`~werkzeug.exceptions.BadRequest`. + + :param e: If parsing failed, this is the exception. It will be + ``None`` if the content type wasn't ``application/json``. + + .. versionchanged:: 2.3 + Raise a 415 error instead of 400. + """ + if e is not None: + raise BadRequest(f"Failed to decode JSON object: {e}") + + raise UnsupportedMediaType( + "Did not attempt to load JSON data because the request" + " Content-Type was not 'application/json'." + ) diff --git a/lib/python3.12/site-packages/werkzeug/wrappers/response.py b/lib/python3.12/site-packages/werkzeug/wrappers/response.py new file mode 100644 index 0000000000000000000000000000000000000000..e05fa8a129bfdfef65e53942a5e0cb027419da4a --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/wrappers/response.py @@ -0,0 +1,831 @@ +from __future__ import annotations + +import json +import typing as t +from http import HTTPStatus +from urllib.parse import urljoin + +from .._internal import _get_environ +from ..datastructures import Headers +from ..http import generate_etag +from ..http import http_date +from ..http import is_resource_modified +from ..http import parse_etags +from ..http import parse_range_header +from ..http import remove_entity_headers +from ..sansio.response import Response as _SansIOResponse +from ..urls import iri_to_uri +from ..utils import cached_property +from ..wsgi import _RangeWrapper +from ..wsgi import ClosingIterator +from ..wsgi import get_current_url + +if t.TYPE_CHECKING: + from _typeshed.wsgi import StartResponse + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + from .request import Request + + +def _iter_encoded(iterable: t.Iterable[str | bytes]) -> t.Iterator[bytes]: + for item in iterable: + if isinstance(item, str): + yield item.encode() + else: + yield item + + +class Response(_SansIOResponse): + """Represents an outgoing WSGI HTTP response with body, status, and + headers. Has properties and methods for using the functionality + defined by various HTTP specs. + + The response body is flexible to support different use cases. The + simple form is passing bytes, or a string which will be encoded as + UTF-8. Passing an iterable of bytes or strings makes this a + streaming response. A generator is particularly useful for building + a CSV file in memory or using SSE (Server Sent Events). A file-like + object is also iterable, although the + :func:`~werkzeug.utils.send_file` helper should be used in that + case. + + The response object is itself a WSGI application callable. When + called (:meth:`__call__`) with ``environ`` and ``start_response``, + it will pass its status and headers to ``start_response`` then + return its body as an iterable. + + .. code-block:: python + + from werkzeug.wrappers.response import Response + + def index(): + return Response("Hello, World!") + + def application(environ, start_response): + path = environ.get("PATH_INFO") or "/" + + if path == "/": + response = index() + else: + response = Response("Not Found", status=404) + + return response(environ, start_response) + + :param response: The data for the body of the response. A string or + bytes, or tuple or list of strings or bytes, for a fixed-length + response, or any other iterable of strings or bytes for a + streaming response. Defaults to an empty body. + :param status: The status code for the response. Either an int, in + which case the default status message is added, or a string in + the form ``{code} {message}``, like ``404 Not Found``. Defaults + to 200. + :param headers: A :class:`~werkzeug.datastructures.Headers` object, + or a list of ``(key, value)`` tuples that will be converted to a + ``Headers`` object. + :param mimetype: The mime type (content type without charset or + other parameters) of the response. If the value starts with + ``text/`` (or matches some other special cases), the charset + will be added to create the ``content_type``. + :param content_type: The full content type of the response. + Overrides building the value from ``mimetype``. + :param direct_passthrough: Pass the response body directly through + as the WSGI iterable. This can be used when the body is a binary + file or other iterator of bytes, to skip some unnecessary + checks. Use :func:`~werkzeug.utils.send_file` instead of setting + this manually. + + .. versionchanged:: 2.1 + Old ``BaseResponse`` and mixin classes were removed. + + .. versionchanged:: 2.0 + Combine ``BaseResponse`` and mixins into a single ``Response`` + class. + + .. versionchanged:: 0.5 + The ``direct_passthrough`` parameter was added. + """ + + #: if set to `False` accessing properties on the response object will + #: not try to consume the response iterator and convert it into a list. + #: + #: .. versionadded:: 0.6.2 + #: + #: That attribute was previously called `implicit_seqence_conversion`. + #: (Notice the typo). If you did use this feature, you have to adapt + #: your code to the name change. + implicit_sequence_conversion = True + + #: If a redirect ``Location`` header is a relative URL, make it an + #: absolute URL, including scheme and domain. + #: + #: .. versionchanged:: 2.1 + #: This is disabled by default, so responses will send relative + #: redirects. + #: + #: .. versionadded:: 0.8 + autocorrect_location_header = False + + #: Should this response object automatically set the content-length + #: header if possible? This is true by default. + #: + #: .. versionadded:: 0.8 + automatically_set_content_length = True + + #: The response body to send as the WSGI iterable. A list of strings + #: or bytes represents a fixed-length response, any other iterable + #: is a streaming response. Strings are encoded to bytes as UTF-8. + #: + #: Do not set to a plain string or bytes, that will cause sending + #: the response to be very inefficient as it will iterate one byte + #: at a time. + response: t.Iterable[str] | t.Iterable[bytes] + + def __init__( + self, + response: t.Iterable[bytes] | bytes | t.Iterable[str] | str | None = None, + status: int | str | HTTPStatus | None = None, + headers: t.Mapping[str, str | t.Iterable[str]] + | t.Iterable[tuple[str, str]] + | None = None, + mimetype: str | None = None, + content_type: str | None = None, + direct_passthrough: bool = False, + ) -> None: + super().__init__( + status=status, + headers=headers, + mimetype=mimetype, + content_type=content_type, + ) + + #: Pass the response body directly through as the WSGI iterable. + #: This can be used when the body is a binary file or other + #: iterator of bytes, to skip some unnecessary checks. Use + #: :func:`~werkzeug.utils.send_file` instead of setting this + #: manually. + self.direct_passthrough = direct_passthrough + self._on_close: list[t.Callable[[], t.Any]] = [] + + # we set the response after the headers so that if a class changes + # the charset attribute, the data is set in the correct charset. + if response is None: + self.response = [] + elif isinstance(response, (str, bytes, bytearray)): + self.set_data(response) + else: + self.response = response + + def call_on_close(self, func: t.Callable[[], t.Any]) -> t.Callable[[], t.Any]: + """Adds a function to the internal list of functions that should + be called as part of closing down the response. Since 0.7 this + function also returns the function that was passed so that this + can be used as a decorator. + + .. versionadded:: 0.6 + """ + self._on_close.append(func) + return func + + def __repr__(self) -> str: + if self.is_sequence: + body_info = f"{sum(map(len, self.iter_encoded()))} bytes" + else: + body_info = "streamed" if self.is_streamed else "likely-streamed" + return f"<{type(self).__name__} {body_info} [{self.status}]>" + + @classmethod + def force_type( + cls, response: Response, environ: WSGIEnvironment | None = None + ) -> Response: + """Enforce that the WSGI response is a response object of the current + type. Werkzeug will use the :class:`Response` internally in many + situations like the exceptions. If you call :meth:`get_response` on an + exception you will get back a regular :class:`Response` object, even + if you are using a custom subclass. + + This method can enforce a given response type, and it will also + convert arbitrary WSGI callables into response objects if an environ + is provided:: + + # convert a Werkzeug response object into an instance of the + # MyResponseClass subclass. + response = MyResponseClass.force_type(response) + + # convert any WSGI application into a response object + response = MyResponseClass.force_type(response, environ) + + This is especially useful if you want to post-process responses in + the main dispatcher and use functionality provided by your subclass. + + Keep in mind that this will modify response objects in place if + possible! + + :param response: a response object or wsgi application. + :param environ: a WSGI environment object. + :return: a response object. + """ + if not isinstance(response, Response): + if environ is None: + raise TypeError( + "cannot convert WSGI application into response" + " objects without an environ" + ) + + from ..test import run_wsgi_app + + response = Response(*run_wsgi_app(response, environ)) + + response.__class__ = cls + return response + + @classmethod + def from_app( + cls, app: WSGIApplication, environ: WSGIEnvironment, buffered: bool = False + ) -> Response: + """Create a new response object from an application output. This + works best if you pass it an application that returns a generator all + the time. Sometimes applications may use the `write()` callable + returned by the `start_response` function. This tries to resolve such + edge cases automatically. But if you don't get the expected output + you should set `buffered` to `True` which enforces buffering. + + :param app: the WSGI application to execute. + :param environ: the WSGI environment to execute against. + :param buffered: set to `True` to enforce buffering. + :return: a response object. + """ + from ..test import run_wsgi_app + + return cls(*run_wsgi_app(app, environ, buffered)) + + @t.overload + def get_data(self, as_text: t.Literal[False] = False) -> bytes: ... + + @t.overload + def get_data(self, as_text: t.Literal[True]) -> str: ... + + def get_data(self, as_text: bool = False) -> bytes | str: + """The string representation of the response body. Whenever you call + this property the response iterable is encoded and flattened. This + can lead to unwanted behavior if you stream big data. + + This behavior can be disabled by setting + :attr:`implicit_sequence_conversion` to `False`. + + If `as_text` is set to `True` the return value will be a decoded + string. + + .. versionadded:: 0.9 + """ + self._ensure_sequence() + rv = b"".join(self.iter_encoded()) + + if as_text: + return rv.decode() + + return rv + + def set_data(self, value: bytes | str) -> None: + """Sets a new string as response. The value must be a string or + bytes. If a string is set it's encoded to the charset of the + response (utf-8 by default). + + .. versionadded:: 0.9 + """ + if isinstance(value, str): + value = value.encode() + self.response = [value] + if self.automatically_set_content_length: + self.headers["Content-Length"] = str(len(value)) + + data = property( + get_data, + set_data, + doc="A descriptor that calls :meth:`get_data` and :meth:`set_data`.", + ) + + def calculate_content_length(self) -> int | None: + """Returns the content length if available or `None` otherwise.""" + try: + self._ensure_sequence() + except RuntimeError: + return None + return sum(len(x) for x in self.iter_encoded()) + + def _ensure_sequence(self, mutable: bool = False) -> None: + """This method can be called by methods that need a sequence. If + `mutable` is true, it will also ensure that the response sequence + is a standard Python list. + + .. versionadded:: 0.6 + """ + if self.is_sequence: + # if we need a mutable object, we ensure it's a list. + if mutable and not isinstance(self.response, list): + self.response = list(self.response) # type: ignore + return + if self.direct_passthrough: + raise RuntimeError( + "Attempted implicit sequence conversion but the" + " response object is in direct passthrough mode." + ) + if not self.implicit_sequence_conversion: + raise RuntimeError( + "The response object required the iterable to be a" + " sequence, but the implicit conversion was disabled." + " Call make_sequence() yourself." + ) + self.make_sequence() + + def make_sequence(self) -> None: + """Converts the response iterator in a list. By default this happens + automatically if required. If `implicit_sequence_conversion` is + disabled, this method is not automatically called and some properties + might raise exceptions. This also encodes all the items. + + .. versionadded:: 0.6 + """ + if not self.is_sequence: + # if we consume an iterable we have to ensure that the close + # method of the iterable is called if available when we tear + # down the response + close = getattr(self.response, "close", None) + self.response = list(self.iter_encoded()) + if close is not None: + self.call_on_close(close) + + def iter_encoded(self) -> t.Iterator[bytes]: + """Iter the response encoded with the encoding of the response. + If the response object is invoked as WSGI application the return + value of this method is used as application iterator unless + :attr:`direct_passthrough` was activated. + """ + # Encode in a separate function so that self.response is fetched + # early. This allows us to wrap the response with the return + # value from get_app_iter or iter_encoded. + return _iter_encoded(self.response) + + @property + def is_streamed(self) -> bool: + """If the response is streamed (the response is not an iterable with + a length information) this property is `True`. In this case streamed + means that there is no information about the number of iterations. + This is usually `True` if a generator is passed to the response object. + + This is useful for checking before applying some sort of post + filtering that should not take place for streamed responses. + """ + try: + len(self.response) # type: ignore + except (TypeError, AttributeError): + return True + return False + + @property + def is_sequence(self) -> bool: + """If the iterator is buffered, this property will be `True`. A + response object will consider an iterator to be buffered if the + response attribute is a list or tuple. + + .. versionadded:: 0.6 + """ + return isinstance(self.response, (tuple, list)) + + def close(self) -> None: + """Close the wrapped response if possible. You can also use the object + in a with statement which will automatically close it. + + .. versionadded:: 0.9 + Can now be used in a with statement. + """ + if hasattr(self.response, "close"): + self.response.close() + for func in self._on_close: + func() + + def __enter__(self) -> Response: + return self + + def __exit__(self, exc_type, exc_value, tb): # type: ignore + self.close() + + def freeze(self) -> None: + """Make the response object ready to be pickled. Does the + following: + + * Buffer the response into a list, ignoring + :attr:`implicity_sequence_conversion` and + :attr:`direct_passthrough`. + * Set the ``Content-Length`` header. + * Generate an ``ETag`` header if one is not already set. + + .. versionchanged:: 2.1 + Removed the ``no_etag`` parameter. + + .. versionchanged:: 2.0 + An ``ETag`` header is always added. + + .. versionchanged:: 0.6 + The ``Content-Length`` header is set. + """ + # Always freeze the encoded response body, ignore + # implicit_sequence_conversion and direct_passthrough. + self.response = list(self.iter_encoded()) + self.headers["Content-Length"] = str(sum(map(len, self.response))) + self.add_etag() + + def get_wsgi_headers(self, environ: WSGIEnvironment) -> Headers: + """This is automatically called right before the response is started + and returns headers modified for the given environment. It returns a + copy of the headers from the response with some modifications applied + if necessary. + + For example the location header (if present) is joined with the root + URL of the environment. Also the content length is automatically set + to zero here for certain status codes. + + .. versionchanged:: 0.6 + Previously that function was called `fix_headers` and modified + the response object in place. Also since 0.6, IRIs in location + and content-location headers are handled properly. + + Also starting with 0.6, Werkzeug will attempt to set the content + length if it is able to figure it out on its own. This is the + case if all the strings in the response iterable are already + encoded and the iterable is buffered. + + :param environ: the WSGI environment of the request. + :return: returns a new :class:`~werkzeug.datastructures.Headers` + object. + """ + headers = Headers(self.headers) + location: str | None = None + content_location: str | None = None + content_length: str | int | None = None + status = self.status_code + + # iterate over the headers to find all values in one go. Because + # get_wsgi_headers is used each response that gives us a tiny + # speedup. + for key, value in headers: + ikey = key.lower() + if ikey == "location": + location = value + elif ikey == "content-location": + content_location = value + elif ikey == "content-length": + content_length = value + + if location is not None: + location = iri_to_uri(location) + + if self.autocorrect_location_header: + # Make the location header an absolute URL. + current_url = get_current_url(environ, strip_querystring=True) + current_url = iri_to_uri(current_url) + location = urljoin(current_url, location) + + headers["Location"] = location + + # make sure the content location is a URL + if content_location is not None: + headers["Content-Location"] = iri_to_uri(content_location) + + if 100 <= status < 200 or status == 204: + # Per section 3.3.2 of RFC 7230, "a server MUST NOT send a + # Content-Length header field in any response with a status + # code of 1xx (Informational) or 204 (No Content)." + headers.remove("Content-Length") + elif status == 304: + remove_entity_headers(headers) + + # if we can determine the content length automatically, we + # should try to do that. But only if this does not involve + # flattening the iterator or encoding of strings in the + # response. We however should not do that if we have a 304 + # response. + if ( + self.automatically_set_content_length + and self.is_sequence + and content_length is None + and status not in (204, 304) + and not (100 <= status < 200) + ): + content_length = sum(len(x) for x in self.iter_encoded()) + headers["Content-Length"] = str(content_length) + + return headers + + def get_app_iter(self, environ: WSGIEnvironment) -> t.Iterable[bytes]: + """Returns the application iterator for the given environ. Depending + on the request method and the current status code the return value + might be an empty response rather than the one from the response. + + If the request method is `HEAD` or the status code is in a range + where the HTTP specification requires an empty response, an empty + iterable is returned. + + .. versionadded:: 0.6 + + :param environ: the WSGI environment of the request. + :return: a response iterable. + """ + status = self.status_code + if ( + environ["REQUEST_METHOD"] == "HEAD" + or 100 <= status < 200 + or status in (204, 304) + ): + iterable: t.Iterable[bytes] = () + elif self.direct_passthrough: + return self.response # type: ignore + else: + iterable = self.iter_encoded() + return ClosingIterator(iterable, self.close) + + def get_wsgi_response( + self, environ: WSGIEnvironment + ) -> tuple[t.Iterable[bytes], str, list[tuple[str, str]]]: + """Returns the final WSGI response as tuple. The first item in + the tuple is the application iterator, the second the status and + the third the list of headers. The response returned is created + specially for the given environment. For example if the request + method in the WSGI environment is ``'HEAD'`` the response will + be empty and only the headers and status code will be present. + + .. versionadded:: 0.6 + + :param environ: the WSGI environment of the request. + :return: an ``(app_iter, status, headers)`` tuple. + """ + headers = self.get_wsgi_headers(environ) + app_iter = self.get_app_iter(environ) + return app_iter, self.status, headers.to_wsgi_list() + + def __call__( + self, environ: WSGIEnvironment, start_response: StartResponse + ) -> t.Iterable[bytes]: + """Process this response as WSGI application. + + :param environ: the WSGI environment. + :param start_response: the response callable provided by the WSGI + server. + :return: an application iterator + """ + app_iter, status, headers = self.get_wsgi_response(environ) + start_response(status, headers) + return app_iter + + # JSON + + #: A module or other object that has ``dumps`` and ``loads`` + #: functions that match the API of the built-in :mod:`json` module. + json_module = json + + @property + def json(self) -> t.Any | None: + """The parsed JSON data if :attr:`mimetype` indicates JSON + (:mimetype:`application/json`, see :attr:`is_json`). + + Calls :meth:`get_json` with default arguments. + """ + return self.get_json() + + @t.overload + def get_json(self, force: bool = ..., silent: t.Literal[False] = ...) -> t.Any: ... + + @t.overload + def get_json(self, force: bool = ..., silent: bool = ...) -> t.Any | None: ... + + def get_json(self, force: bool = False, silent: bool = False) -> t.Any | None: + """Parse :attr:`data` as JSON. Useful during testing. + + If the mimetype does not indicate JSON + (:mimetype:`application/json`, see :attr:`is_json`), this + returns ``None``. + + Unlike :meth:`Request.get_json`, the result is not cached. + + :param force: Ignore the mimetype and always try to parse JSON. + :param silent: Silence parsing errors and return ``None`` + instead. + """ + if not (force or self.is_json): + return None + + data = self.get_data() + + try: + return self.json_module.loads(data) + except ValueError: + if not silent: + raise + + return None + + # Stream + + @cached_property + def stream(self) -> ResponseStream: + """The response iterable as write-only stream.""" + return ResponseStream(self) + + def _wrap_range_response(self, start: int, length: int) -> None: + """Wrap existing Response in case of Range Request context.""" + if self.status_code == 206: + self.response = _RangeWrapper(self.response, start, length) # type: ignore + + def _is_range_request_processable(self, environ: WSGIEnvironment) -> bool: + """Return ``True`` if `Range` header is present and if underlying + resource is considered unchanged when compared with `If-Range` header. + """ + return ( + "HTTP_IF_RANGE" not in environ + or not is_resource_modified( + environ, + self.headers.get("etag"), + None, + self.headers.get("last-modified"), + ignore_if_range=False, + ) + ) and "HTTP_RANGE" in environ + + def _process_range_request( + self, + environ: WSGIEnvironment, + complete_length: int | None, + accept_ranges: bool | str, + ) -> bool: + """Handle Range Request related headers (RFC7233). If `Accept-Ranges` + header is valid, and Range Request is processable, we set the headers + as described by the RFC, and wrap the underlying response in a + RangeWrapper. + + Returns ``True`` if Range Request can be fulfilled, ``False`` otherwise. + + :raises: :class:`~werkzeug.exceptions.RequestedRangeNotSatisfiable` + if `Range` header could not be parsed or satisfied. + + .. versionchanged:: 2.0 + Returns ``False`` if the length is 0. + """ + from ..exceptions import RequestedRangeNotSatisfiable + + if ( + not accept_ranges + or complete_length is None + or complete_length == 0 + or not self._is_range_request_processable(environ) + ): + return False + + if accept_ranges is True: + accept_ranges = "bytes" + + parsed_range = parse_range_header(environ.get("HTTP_RANGE")) + + if parsed_range is None: + raise RequestedRangeNotSatisfiable(complete_length) + + range_tuple = parsed_range.range_for_length(complete_length) + content_range_header = parsed_range.to_content_range_header(complete_length) + + if range_tuple is None or content_range_header is None: + raise RequestedRangeNotSatisfiable(complete_length) + + content_length = range_tuple[1] - range_tuple[0] + self.headers["Content-Length"] = str(content_length) + self.headers["Accept-Ranges"] = accept_ranges + self.content_range = content_range_header + self.status_code = 206 + self._wrap_range_response(range_tuple[0], content_length) + return True + + def make_conditional( + self, + request_or_environ: WSGIEnvironment | Request, + accept_ranges: bool | str = False, + complete_length: int | None = None, + ) -> Response: + """Make the response conditional to the request. This method works + best if an etag was defined for the response already. The `add_etag` + method can be used to do that. If called without etag just the date + header is set. + + This does nothing if the request method in the request or environ is + anything but GET or HEAD. + + For optimal performance when handling range requests, it's recommended + that your response data object implements `seekable`, `seek` and `tell` + methods as described by :py:class:`io.IOBase`. Objects returned by + :meth:`~werkzeug.wsgi.wrap_file` automatically implement those methods. + + It does not remove the body of the response because that's something + the :meth:`__call__` function does for us automatically. + + Returns self so that you can do ``return resp.make_conditional(req)`` + but modifies the object in-place. + + :param request_or_environ: a request object or WSGI environment to be + used to make the response conditional + against. + :param accept_ranges: This parameter dictates the value of + `Accept-Ranges` header. If ``False`` (default), + the header is not set. If ``True``, it will be set + to ``"bytes"``. If it's a string, it will use this + value. + :param complete_length: Will be used only in valid Range Requests. + It will set `Content-Range` complete length + value and compute `Content-Length` real value. + This parameter is mandatory for successful + Range Requests completion. + :raises: :class:`~werkzeug.exceptions.RequestedRangeNotSatisfiable` + if `Range` header could not be parsed or satisfied. + + .. versionchanged:: 2.0 + Range processing is skipped if length is 0 instead of + raising a 416 Range Not Satisfiable error. + """ + environ = _get_environ(request_or_environ) + if environ["REQUEST_METHOD"] in ("GET", "HEAD"): + # if the date is not in the headers, add it now. We however + # will not override an already existing header. Unfortunately + # this header will be overridden by many WSGI servers including + # wsgiref. + if "date" not in self.headers: + self.headers["Date"] = http_date() + is206 = self._process_range_request(environ, complete_length, accept_ranges) + if not is206 and not is_resource_modified( + environ, + self.headers.get("etag"), + None, + self.headers.get("last-modified"), + ): + if parse_etags(environ.get("HTTP_IF_MATCH")): + self.status_code = 412 + else: + self.status_code = 304 + if ( + self.automatically_set_content_length + and "content-length" not in self.headers + ): + length = self.calculate_content_length() + if length is not None: + self.headers["Content-Length"] = str(length) + return self + + def add_etag(self, overwrite: bool = False, weak: bool = False) -> None: + """Add an etag for the current response if there is none yet. + + .. versionchanged:: 2.0 + SHA-1 is used to generate the value. MD5 may not be + available in some environments. + """ + if overwrite or "etag" not in self.headers: + self.set_etag(generate_etag(self.get_data()), weak) + + +class ResponseStream: + """A file descriptor like object used by :meth:`Response.stream` to + represent the body of the stream. It directly pushes into the + response iterable of the response object. + """ + + mode = "wb+" + + def __init__(self, response: Response): + self.response = response + self.closed = False + + def write(self, value: bytes) -> int: + if self.closed: + raise ValueError("I/O operation on closed file") + self.response._ensure_sequence(mutable=True) + self.response.response.append(value) # type: ignore + self.response.headers.pop("Content-Length", None) + return len(value) + + def writelines(self, seq: t.Iterable[bytes]) -> None: + for item in seq: + self.write(item) + + def close(self) -> None: + self.closed = True + + def flush(self) -> None: + if self.closed: + raise ValueError("I/O operation on closed file") + + def isatty(self) -> bool: + if self.closed: + raise ValueError("I/O operation on closed file") + return False + + def tell(self) -> int: + self.response._ensure_sequence() + return sum(map(len, self.response.response)) + + @property + def encoding(self) -> str: + return "utf-8" diff --git a/lib/python3.12/site-packages/werkzeug/wsgi.py b/lib/python3.12/site-packages/werkzeug/wsgi.py new file mode 100644 index 0000000000000000000000000000000000000000..2d071c789e83e6d5999ae10074f7a2c5d827cbc6 --- /dev/null +++ b/lib/python3.12/site-packages/werkzeug/wsgi.py @@ -0,0 +1,595 @@ +from __future__ import annotations + +import io +import typing as t +from functools import partial +from functools import update_wrapper + +from .exceptions import ClientDisconnected +from .exceptions import RequestEntityTooLarge +from .sansio import utils as _sansio_utils +from .sansio.utils import host_is_trusted # noqa: F401 # Imported as part of API + +if t.TYPE_CHECKING: + from _typeshed.wsgi import WSGIApplication + from _typeshed.wsgi import WSGIEnvironment + + +def responder(f: t.Callable[..., WSGIApplication]) -> WSGIApplication: + """Marks a function as responder. Decorate a function with it and it + will automatically call the return value as WSGI application. + + Example:: + + @responder + def application(environ, start_response): + return Response('Hello World!') + """ + return update_wrapper(lambda *a: f(*a)(*a[-2:]), f) + + +def get_current_url( + environ: WSGIEnvironment, + root_only: bool = False, + strip_querystring: bool = False, + host_only: bool = False, + trusted_hosts: t.Iterable[str] | None = None, +) -> str: + """Recreate the URL for a request from the parts in a WSGI + environment. + + The URL is an IRI, not a URI, so it may contain Unicode characters. + Use :func:`~werkzeug.urls.iri_to_uri` to convert it to ASCII. + + :param environ: The WSGI environment to get the URL parts from. + :param root_only: Only build the root path, don't include the + remaining path or query string. + :param strip_querystring: Don't include the query string. + :param host_only: Only build the scheme and host. + :param trusted_hosts: A list of trusted host names to validate the + host against. + """ + parts = { + "scheme": environ["wsgi.url_scheme"], + "host": get_host(environ, trusted_hosts), + } + + if not host_only: + parts["root_path"] = environ.get("SCRIPT_NAME", "") + + if not root_only: + parts["path"] = environ.get("PATH_INFO", "") + + if not strip_querystring: + parts["query_string"] = environ.get("QUERY_STRING", "").encode("latin1") + + return _sansio_utils.get_current_url(**parts) + + +def _get_server( + environ: WSGIEnvironment, +) -> tuple[str, int | None] | None: + name = environ.get("SERVER_NAME") + + if name is None: + return None + + try: + port: int | None = int(environ.get("SERVER_PORT", None)) # type: ignore[arg-type] + except (TypeError, ValueError): + # unix socket + port = None + + return name, port + + +def get_host( + environ: WSGIEnvironment, trusted_hosts: t.Iterable[str] | None = None +) -> str: + """Return the host for the given WSGI environment. + + The ``Host`` header is preferred, then ``SERVER_NAME`` if it's not + set. The returned host will only contain the port if it is different + than the standard port for the protocol. + + Optionally, verify that the host is trusted using + :func:`host_is_trusted` and raise a + :exc:`~werkzeug.exceptions.SecurityError` if it is not. + + :param environ: A WSGI environment dict. + :param trusted_hosts: A list of trusted host names. + + :return: Host, with port if necessary. + :raise ~werkzeug.exceptions.SecurityError: If the host is not + trusted. + """ + return _sansio_utils.get_host( + environ["wsgi.url_scheme"], + environ.get("HTTP_HOST"), + _get_server(environ), + trusted_hosts, + ) + + +def get_content_length(environ: WSGIEnvironment) -> int | None: + """Return the ``Content-Length`` header value as an int. If the header is not given + or the ``Transfer-Encoding`` header is ``chunked``, ``None`` is returned to indicate + a streaming request. If the value is not an integer, or negative, 0 is returned. + + :param environ: The WSGI environ to get the content length from. + + .. versionadded:: 0.9 + """ + return _sansio_utils.get_content_length( + http_content_length=environ.get("CONTENT_LENGTH"), + http_transfer_encoding=environ.get("HTTP_TRANSFER_ENCODING"), + ) + + +def get_input_stream( + environ: WSGIEnvironment, + safe_fallback: bool = True, + max_content_length: int | None = None, +) -> t.IO[bytes]: + """Return the WSGI input stream, wrapped so that it may be read safely without going + past the ``Content-Length`` header value or ``max_content_length``. + + If ``Content-Length`` exceeds ``max_content_length``, a + :exc:`RequestEntityTooLarge`` ``413 Content Too Large`` error is raised. + + If the WSGI server sets ``environ["wsgi.input_terminated"]``, it indicates that the + server handles terminating the stream, so it is safe to read directly. For example, + a server that knows how to handle chunked requests safely would set this. + + If ``max_content_length`` is set, it can be enforced on streams if + ``wsgi.input_terminated`` is set. Otherwise, an empty stream is returned unless the + user explicitly disables this safe fallback. + + If the limit is reached before the underlying stream is exhausted (such as a file + that is too large, or an infinite stream), the remaining contents of the stream + cannot be read safely. Depending on how the server handles this, clients may show a + "connection reset" failure instead of seeing the 413 response. + + :param environ: The WSGI environ containing the stream. + :param safe_fallback: Return an empty stream when ``Content-Length`` is not set. + Disabling this allows infinite streams, which can be a denial-of-service risk. + :param max_content_length: The maximum length that content-length or streaming + requests may not exceed. + + .. versionchanged:: 2.3.2 + ``max_content_length`` is only applied to streaming requests if the server sets + ``wsgi.input_terminated``. + + .. versionchanged:: 2.3 + Check ``max_content_length`` and raise an error if it is exceeded. + + .. versionadded:: 0.9 + """ + stream = t.cast(t.IO[bytes], environ["wsgi.input"]) + content_length = get_content_length(environ) + + if content_length is not None and max_content_length is not None: + if content_length > max_content_length: + raise RequestEntityTooLarge() + + # A WSGI server can set this to indicate that it terminates the input stream. In + # that case the stream is safe without wrapping, or can enforce a max length. + if "wsgi.input_terminated" in environ: + if max_content_length is not None: + # If this is moved above, it can cause the stream to hang if a read attempt + # is made when the client sends no data. For example, the development server + # does not handle buffering except for chunked encoding. + return t.cast( + t.IO[bytes], LimitedStream(stream, max_content_length, is_max=True) + ) + + return stream + + # No limit given, return an empty stream unless the user explicitly allows the + # potentially infinite stream. An infinite stream is dangerous if it's not expected, + # as it can tie up a worker indefinitely. + if content_length is None: + return io.BytesIO() if safe_fallback else stream + + return t.cast(t.IO[bytes], LimitedStream(stream, content_length)) + + +def get_path_info(environ: WSGIEnvironment) -> str: + """Return ``PATH_INFO`` from the WSGI environment. + + :param environ: WSGI environment to get the path from. + + .. versionchanged:: 3.0 + The ``charset`` and ``errors`` parameters were removed. + + .. versionadded:: 0.9 + """ + path: bytes = environ.get("PATH_INFO", "").encode("latin1") + return path.decode(errors="replace") + + +class ClosingIterator: + """The WSGI specification requires that all middlewares and gateways + respect the `close` callback of the iterable returned by the application. + Because it is useful to add another close action to a returned iterable + and adding a custom iterable is a boring task this class can be used for + that:: + + return ClosingIterator(app(environ, start_response), [cleanup_session, + cleanup_locals]) + + If there is just one close function it can be passed instead of the list. + + A closing iterator is not needed if the application uses response objects + and finishes the processing if the response is started:: + + try: + return response(environ, start_response) + finally: + cleanup_session() + cleanup_locals() + """ + + def __init__( + self, + iterable: t.Iterable[bytes], + callbacks: None + | (t.Callable[[], None] | t.Iterable[t.Callable[[], None]]) = None, + ) -> None: + iterator = iter(iterable) + self._next = t.cast(t.Callable[[], bytes], partial(next, iterator)) + if callbacks is None: + callbacks = [] + elif callable(callbacks): + callbacks = [callbacks] + else: + callbacks = list(callbacks) + iterable_close = getattr(iterable, "close", None) + if iterable_close: + callbacks.insert(0, iterable_close) + self._callbacks = callbacks + + def __iter__(self) -> ClosingIterator: + return self + + def __next__(self) -> bytes: + return self._next() + + def close(self) -> None: + for callback in self._callbacks: + callback() + + +def wrap_file( + environ: WSGIEnvironment, file: t.IO[bytes], buffer_size: int = 8192 +) -> t.Iterable[bytes]: + """Wraps a file. This uses the WSGI server's file wrapper if available + or otherwise the generic :class:`FileWrapper`. + + .. versionadded:: 0.5 + + If the file wrapper from the WSGI server is used it's important to not + iterate over it from inside the application but to pass it through + unchanged. If you want to pass out a file wrapper inside a response + object you have to set :attr:`Response.direct_passthrough` to `True`. + + More information about file wrappers are available in :pep:`333`. + + :param file: a :class:`file`-like object with a :meth:`~file.read` method. + :param buffer_size: number of bytes for one iteration. + """ + return environ.get("wsgi.file_wrapper", FileWrapper)( # type: ignore + file, buffer_size + ) + + +class FileWrapper: + """This class can be used to convert a :class:`file`-like object into + an iterable. It yields `buffer_size` blocks until the file is fully + read. + + You should not use this class directly but rather use the + :func:`wrap_file` function that uses the WSGI server's file wrapper + support if it's available. + + .. versionadded:: 0.5 + + If you're using this object together with a :class:`Response` you have + to use the `direct_passthrough` mode. + + :param file: a :class:`file`-like object with a :meth:`~file.read` method. + :param buffer_size: number of bytes for one iteration. + """ + + def __init__(self, file: t.IO[bytes], buffer_size: int = 8192) -> None: + self.file = file + self.buffer_size = buffer_size + + def close(self) -> None: + if hasattr(self.file, "close"): + self.file.close() + + def seekable(self) -> bool: + if hasattr(self.file, "seekable"): + return self.file.seekable() + if hasattr(self.file, "seek"): + return True + return False + + def seek(self, *args: t.Any) -> None: + if hasattr(self.file, "seek"): + self.file.seek(*args) + + def tell(self) -> int | None: + if hasattr(self.file, "tell"): + return self.file.tell() + return None + + def __iter__(self) -> FileWrapper: + return self + + def __next__(self) -> bytes: + data = self.file.read(self.buffer_size) + if data: + return data + raise StopIteration() + + +class _RangeWrapper: + # private for now, but should we make it public in the future ? + + """This class can be used to convert an iterable object into + an iterable that will only yield a piece of the underlying content. + It yields blocks until the underlying stream range is fully read. + The yielded blocks will have a size that can't exceed the original + iterator defined block size, but that can be smaller. + + If you're using this object together with a :class:`Response` you have + to use the `direct_passthrough` mode. + + :param iterable: an iterable object with a :meth:`__next__` method. + :param start_byte: byte from which read will start. + :param byte_range: how many bytes to read. + """ + + def __init__( + self, + iterable: t.Iterable[bytes] | t.IO[bytes], + start_byte: int = 0, + byte_range: int | None = None, + ): + self.iterable = iter(iterable) + self.byte_range = byte_range + self.start_byte = start_byte + self.end_byte = None + + if byte_range is not None: + self.end_byte = start_byte + byte_range + + self.read_length = 0 + self.seekable = hasattr(iterable, "seekable") and iterable.seekable() + self.end_reached = False + + def __iter__(self) -> _RangeWrapper: + return self + + def _next_chunk(self) -> bytes: + try: + chunk = next(self.iterable) + self.read_length += len(chunk) + return chunk + except StopIteration: + self.end_reached = True + raise + + def _first_iteration(self) -> tuple[bytes | None, int]: + chunk = None + if self.seekable: + self.iterable.seek(self.start_byte) # type: ignore + self.read_length = self.iterable.tell() # type: ignore + contextual_read_length = self.read_length + else: + while self.read_length <= self.start_byte: + chunk = self._next_chunk() + if chunk is not None: + chunk = chunk[self.start_byte - self.read_length :] + contextual_read_length = self.start_byte + return chunk, contextual_read_length + + def _next(self) -> bytes: + if self.end_reached: + raise StopIteration() + chunk = None + contextual_read_length = self.read_length + if self.read_length == 0: + chunk, contextual_read_length = self._first_iteration() + if chunk is None: + chunk = self._next_chunk() + if self.end_byte is not None and self.read_length >= self.end_byte: + self.end_reached = True + return chunk[: self.end_byte - contextual_read_length] + return chunk + + def __next__(self) -> bytes: + chunk = self._next() + if chunk: + return chunk + self.end_reached = True + raise StopIteration() + + def close(self) -> None: + if hasattr(self.iterable, "close"): + self.iterable.close() + + +class LimitedStream(io.RawIOBase): + """Wrap a stream so that it doesn't read more than a given limit. This is used to + limit ``wsgi.input`` to the ``Content-Length`` header value or + :attr:`.Request.max_content_length`. + + When attempting to read after the limit has been reached, :meth:`on_exhausted` is + called. When the limit is a maximum, this raises :exc:`.RequestEntityTooLarge`. + + If reading from the stream returns zero bytes or raises an error, + :meth:`on_disconnect` is called, which raises :exc:`.ClientDisconnected`. When the + limit is a maximum and zero bytes were read, no error is raised, since it may be the + end of the stream. + + If the limit is reached before the underlying stream is exhausted (such as a file + that is too large, or an infinite stream), the remaining contents of the stream + cannot be read safely. Depending on how the server handles this, clients may show a + "connection reset" failure instead of seeing the 413 response. + + :param stream: The stream to read from. Must be a readable binary IO object. + :param limit: The limit in bytes to not read past. Should be either the + ``Content-Length`` header value or ``request.max_content_length``. + :param is_max: Whether the given ``limit`` is ``request.max_content_length`` instead + of the ``Content-Length`` header value. This changes how exhausted and + disconnect events are handled. + + .. versionchanged:: 2.3 + Handle ``max_content_length`` differently than ``Content-Length``. + + .. versionchanged:: 2.3 + Implements ``io.RawIOBase`` rather than ``io.IOBase``. + """ + + def __init__(self, stream: t.IO[bytes], limit: int, is_max: bool = False) -> None: + self._stream = stream + self._pos = 0 + self.limit = limit + self._limit_is_max = is_max + + @property + def is_exhausted(self) -> bool: + """Whether the current stream position has reached the limit.""" + return self._pos >= self.limit + + def on_exhausted(self) -> None: + """Called when attempting to read after the limit has been reached. + + The default behavior is to do nothing, unless the limit is a maximum, in which + case it raises :exc:`.RequestEntityTooLarge`. + + .. versionchanged:: 2.3 + Raises ``RequestEntityTooLarge`` if the limit is a maximum. + + .. versionchanged:: 2.3 + Any return value is ignored. + """ + if self._limit_is_max: + raise RequestEntityTooLarge() + + def on_disconnect(self, error: Exception | None = None) -> None: + """Called when an attempted read receives zero bytes before the limit was + reached. This indicates that the client disconnected before sending the full + request body. + + The default behavior is to raise :exc:`.ClientDisconnected`, unless the limit is + a maximum and no error was raised. + + .. versionchanged:: 2.3 + Added the ``error`` parameter. Do nothing if the limit is a maximum and no + error was raised. + + .. versionchanged:: 2.3 + Any return value is ignored. + """ + if not self._limit_is_max or error is not None: + raise ClientDisconnected() + + # If the limit is a maximum, then we may have read zero bytes because the + # streaming body is complete. There's no way to distinguish that from the + # client disconnecting early. + + def exhaust(self) -> bytes: + """Exhaust the stream by reading until the limit is reached or the client + disconnects, returning the remaining data. + + .. versionchanged:: 2.3 + Return the remaining data. + + .. versionchanged:: 2.2.3 + Handle case where wrapped stream returns fewer bytes than requested. + """ + if not self.is_exhausted: + return self.readall() + + return b"" + + def readinto(self, b: bytearray) -> int | None: # type: ignore[override] + size = len(b) + remaining = self.limit - self._pos + + if remaining <= 0: + self.on_exhausted() + return 0 + + if hasattr(self._stream, "readinto"): + # Use stream.readinto if it's available. + if size <= remaining: + # The size fits in the remaining limit, use the buffer directly. + try: + out_size: int | None = self._stream.readinto(b) + except (OSError, ValueError) as e: + self.on_disconnect(error=e) + return 0 + else: + # Use a temp buffer with the remaining limit as the size. + temp_b = bytearray(remaining) + + try: + out_size = self._stream.readinto(temp_b) + except (OSError, ValueError) as e: + self.on_disconnect(error=e) + return 0 + + if out_size: + b[:out_size] = temp_b + else: + # WSGI requires that stream.read is available. + try: + data = self._stream.read(min(size, remaining)) + except (OSError, ValueError) as e: + self.on_disconnect(error=e) + return 0 + + out_size = len(data) + b[:out_size] = data + + if not out_size: + # Read zero bytes from the stream. + self.on_disconnect() + return 0 + + self._pos += out_size + return out_size + + def readall(self) -> bytes: + if self.is_exhausted: + self.on_exhausted() + return b"" + + out = bytearray() + + # The parent implementation uses "while True", which results in an extra read. + while not self.is_exhausted: + data = self.read(1024 * 64) + + # Stream may return empty before a max limit is reached. + if not data: + break + + out.extend(data) + + return bytes(out) + + def tell(self) -> int: + """Return the current stream position. + + .. versionadded:: 0.9 + """ + return self._pos + + def readable(self) -> bool: + return True diff --git a/lib/python3.12/site-packages/yarl/__init__.py b/lib/python3.12/site-packages/yarl/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e45554be4aa1babca506934bec6c0800dbe8adaa --- /dev/null +++ b/lib/python3.12/site-packages/yarl/__init__.py @@ -0,0 +1,14 @@ +from ._query import Query, QueryVariable, SimpleQuery +from ._url import URL, cache_clear, cache_configure, cache_info + +__version__ = "1.22.0" + +__all__ = ( + "URL", + "SimpleQuery", + "QueryVariable", + "Query", + "cache_clear", + "cache_configure", + "cache_info", +) diff --git a/lib/python3.12/site-packages/yarl/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f3ea9d5a3792bf959354186cc72e0bbbd66e07ec Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/__pycache__/_parse.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/_parse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..840af7f704d66e58c649f047ce617cd0050c8793 Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/_parse.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/__pycache__/_path.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/_path.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7f430e70d94a0fa42c2e26e213ab166105d08bca Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/_path.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/__pycache__/_query.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/_query.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..eec49009b1c32884c966819bc1e5758dd6ec364d Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/_query.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/__pycache__/_quoters.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/_quoters.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cbb6f09c1190c76b10a3540d5fdad74b9400d3bc Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/_quoters.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/__pycache__/_quoting.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/_quoting.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69d25d7f2efb493d0149cb3c396966dfe0baeee4 Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/_quoting.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/__pycache__/_quoting_py.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/_quoting_py.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..27f5242e6a280fde3fe614e7de816da7c2276625 Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/_quoting_py.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/__pycache__/_url.cpython-312.pyc b/lib/python3.12/site-packages/yarl/__pycache__/_url.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..17dab019ce3dc3ea75c764a074414607c66b7a89 Binary files /dev/null and b/lib/python3.12/site-packages/yarl/__pycache__/_url.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/yarl/_parse.py b/lib/python3.12/site-packages/yarl/_parse.py new file mode 100644 index 0000000000000000000000000000000000000000..115d772360e61f4322eb72ced557d42a30930518 --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_parse.py @@ -0,0 +1,203 @@ +"""URL parsing utilities.""" + +import re +import unicodedata +from functools import lru_cache +from typing import Union +from urllib.parse import scheme_chars, uses_netloc + +from ._quoters import QUOTER, UNQUOTER_PLUS + +# Leading and trailing C0 control and space to be stripped per WHATWG spec. +# == "".join([chr(i) for i in range(0, 0x20 + 1)]) +WHATWG_C0_CONTROL_OR_SPACE = ( + "\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10" + "\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f " +) + +# Unsafe bytes to be removed per WHATWG spec +UNSAFE_URL_BYTES_TO_REMOVE = ["\t", "\r", "\n"] +USES_AUTHORITY = frozenset(uses_netloc) + +SplitURLType = tuple[str, str, str, str, str] + + +def split_url(url: str) -> SplitURLType: + """Split URL into parts.""" + # Adapted from urllib.parse.urlsplit + # Only lstrip url as some applications rely on preserving trailing space. + # (https://url.spec.whatwg.org/#concept-basic-url-parser would strip both) + url = url.lstrip(WHATWG_C0_CONTROL_OR_SPACE) + for b in UNSAFE_URL_BYTES_TO_REMOVE: + if b in url: + url = url.replace(b, "") + + scheme = netloc = query = fragment = "" + i = url.find(":") + if i > 0 and url[0] in scheme_chars: + for c in url[1:i]: + if c not in scheme_chars: + break + else: + scheme, url = url[:i].lower(), url[i + 1 :] + has_hash = "#" in url + has_question_mark = "?" in url + if url[:2] == "//": + delim = len(url) # position of end of domain part of url, default is end + if has_hash and has_question_mark: + delim_chars = "/?#" + elif has_question_mark: + delim_chars = "/?" + elif has_hash: + delim_chars = "/#" + else: + delim_chars = "/" + for c in delim_chars: # look for delimiters; the order is NOT important + wdelim = url.find(c, 2) # find first of this delim + if wdelim >= 0 and wdelim < delim: # if found + delim = wdelim # use earliest delim position + netloc = url[2:delim] + url = url[delim:] + has_left_bracket = "[" in netloc + has_right_bracket = "]" in netloc + if (has_left_bracket and not has_right_bracket) or ( + has_right_bracket and not has_left_bracket + ): + raise ValueError("Invalid IPv6 URL") + if has_left_bracket: + bracketed_host = netloc.partition("[")[2].partition("]")[0] + # Valid bracketed hosts are defined in + # https://www.rfc-editor.org/rfc/rfc3986#page-49 + # https://url.spec.whatwg.org/ + if bracketed_host and bracketed_host[0] == "v": + if not re.match(r"\Av[a-fA-F0-9]+\..+\Z", bracketed_host): + raise ValueError("IPvFuture address is invalid") + elif ":" not in bracketed_host: + raise ValueError("The IPv6 content between brackets is not valid") + if has_hash: + url, _, fragment = url.partition("#") + if has_question_mark: + url, _, query = url.partition("?") + if netloc and not netloc.isascii(): + _check_netloc(netloc) + return scheme, netloc, url, query, fragment + + +def _check_netloc(netloc: str) -> None: + # Adapted from urllib.parse._checknetloc + # looking for characters like \u2100 that expand to 'a/c' + # IDNA uses NFKC equivalence, so normalize for this check + + # ignore characters already included + # but not the surrounding text + n = netloc.replace("@", "").replace(":", "").replace("#", "").replace("?", "") + normalized_netloc = unicodedata.normalize("NFKC", n) + if n == normalized_netloc: + return + # Note that there are no unicode decompositions for the character '@' so + # its currently impossible to have test coverage for this branch, however if the + # one should be added in the future we want to make sure its still checked. + for c in "/?#@:": # pragma: no branch + if c in normalized_netloc: + raise ValueError( + f"netloc '{netloc}' contains invalid " + "characters under NFKC normalization" + ) + + +@lru_cache # match the same size as urlsplit +def split_netloc( + netloc: str, +) -> tuple[Union[str, None], Union[str, None], Union[str, None], Union[int, None]]: + """Split netloc into username, password, host and port.""" + if "@" not in netloc: + username: Union[str, None] = None + password: Union[str, None] = None + hostinfo = netloc + else: + userinfo, _, hostinfo = netloc.rpartition("@") + username, have_password, password = userinfo.partition(":") + if not have_password: + password = None + + if "[" in hostinfo: + _, _, bracketed = hostinfo.partition("[") + hostname, _, port_str = bracketed.partition("]") + _, _, port_str = port_str.partition(":") + else: + hostname, _, port_str = hostinfo.partition(":") + + if not port_str: + return username or None, password, hostname or None, None + + try: + port = int(port_str) + except ValueError: + raise ValueError("Invalid URL: port can't be converted to integer") + if not (0 <= port <= 65535): + raise ValueError("Port out of range 0-65535") + return username or None, password, hostname or None, port + + +def unsplit_result( + scheme: str, netloc: str, url: str, query: str, fragment: str +) -> str: + """Unsplit a URL without any normalization.""" + if netloc or (scheme and scheme in USES_AUTHORITY) or url[:2] == "//": + if url and url[:1] != "/": + url = f"{scheme}://{netloc}/{url}" if scheme else f"{scheme}:{url}" + else: + url = f"{scheme}://{netloc}{url}" if scheme else f"//{netloc}{url}" + elif scheme: + url = f"{scheme}:{url}" + if query: + url = f"{url}?{query}" + return f"{url}#{fragment}" if fragment else url + + +@lru_cache # match the same size as urlsplit +def make_netloc( + user: Union[str, None], + password: Union[str, None], + host: Union[str, None], + port: Union[int, None], + encode: bool = False, +) -> str: + """Make netloc from parts. + + The user and password are encoded if encode is True. + + The host must already be encoded with _encode_host. + """ + if host is None: + return "" + ret = host + if port is not None: + ret = f"{ret}:{port}" + if user is None and password is None: + return ret + if password is not None: + if not user: + user = "" + elif encode: + user = QUOTER(user) + if encode: + password = QUOTER(password) + user = f"{user}:{password}" + elif user and encode: + user = QUOTER(user) + return f"{user}@{ret}" if user else ret + + +def query_to_pairs(query_string: str) -> list[tuple[str, str]]: + """Parse a query given as a string argument. + + Works like urllib.parse.parse_qsl with keep empty values. + """ + pairs: list[tuple[str, str]] = [] + if not query_string: + return pairs + for k_v in query_string.split("&"): + k, _, v = k_v.partition("=") + pairs.append((UNQUOTER_PLUS(k), UNQUOTER_PLUS(v))) + return pairs diff --git a/lib/python3.12/site-packages/yarl/_path.py b/lib/python3.12/site-packages/yarl/_path.py new file mode 100644 index 0000000000000000000000000000000000000000..c22f0b4b8cdd9280fd36789e2bc052b1c4938167 --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_path.py @@ -0,0 +1,41 @@ +"""Utilities for working with paths.""" + +from collections.abc import Sequence +from contextlib import suppress + + +def normalize_path_segments(segments: Sequence[str]) -> list[str]: + """Drop '.' and '..' from a sequence of str segments""" + + resolved_path: list[str] = [] + + for seg in segments: + if seg == "..": + # ignore any .. segments that would otherwise cause an + # IndexError when popped from resolved_path if + # resolving for rfc3986 + with suppress(IndexError): + resolved_path.pop() + elif seg != ".": + resolved_path.append(seg) + + if segments and segments[-1] in (".", ".."): + # do some post-processing here. + # if the last segment was a relative dir, + # then we need to append the trailing '/' + resolved_path.append("") + + return resolved_path + + +def normalize_path(path: str) -> str: + # Drop '.' and '..' from str path + prefix = "" + if path and path[0] == "/": + # preserve the "/" root element of absolute paths, copying it to the + # normalised output as per sections 5.2.4 and 6.2.2.3 of rfc3986. + prefix = "/" + path = path[1:] + + segments = path.split("/") + return prefix + "/".join(normalize_path_segments(segments)) diff --git a/lib/python3.12/site-packages/yarl/_query.py b/lib/python3.12/site-packages/yarl/_query.py new file mode 100644 index 0000000000000000000000000000000000000000..d911bcf0b24a6defa12c306d6342ba699e137b3d --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_query.py @@ -0,0 +1,121 @@ +"""Query string handling.""" + +import math +from collections.abc import Iterable, Mapping, Sequence +from typing import TYPE_CHECKING, Any, SupportsInt, Union, cast + +from multidict import istr + +from ._quoters import QUERY_PART_QUOTER, QUERY_QUOTER + +SimpleQuery = Union[str, SupportsInt, float] +QueryVariable = Union[SimpleQuery, Sequence[SimpleQuery]] +Query = Union[ + None, str, Mapping[str, QueryVariable], Sequence[tuple[str, QueryVariable]] +] + + +def query_var(v: SimpleQuery) -> str: + """Convert a query variable to a string.""" + cls = type(v) + if cls is int: # Fast path for non-subclassed int + return str(v) + if isinstance(v, str): + return v + if isinstance(v, float): + if math.isinf(v): + raise ValueError("float('inf') is not supported") + if math.isnan(v): + raise ValueError("float('nan') is not supported") + return str(float(v)) + if cls is not bool and isinstance(v, SupportsInt): + return str(int(v)) + raise TypeError( + "Invalid variable type: value " + "should be str, int or float, got {!r} " + "of type {}".format(v, cls) + ) + + +def get_str_query_from_sequence_iterable( + items: Iterable[tuple[Union[str, istr], QueryVariable]], +) -> str: + """Return a query string from a sequence of (key, value) pairs. + + value is a single value or a sequence of values for the key + + The sequence of values must be a list or tuple. + """ + quoter = QUERY_PART_QUOTER + pairs = [ + f"{quoter(k)}={quoter(v if type(v) is str else query_var(v))}" + for k, val in items + for v in ( + val if type(val) is not str and isinstance(val, (list, tuple)) else (val,) + ) + ] + return "&".join(pairs) + + +def get_str_query_from_iterable( + items: Iterable[tuple[Union[str, istr], SimpleQuery]], +) -> str: + """Return a query string from an iterable. + + The iterable must contain (key, value) pairs. + + The values are not allowed to be sequences, only single values are + allowed. For sequences, use `_get_str_query_from_sequence_iterable`. + """ + quoter = QUERY_PART_QUOTER + # A listcomp is used since listcomps are inlined on CPython 3.12+ and + # they are a bit faster than a generator expression. + pairs = [ + f"{quoter(k)}={quoter(v if type(v) is str else query_var(v))}" for k, v in items + ] + return "&".join(pairs) + + +def get_str_query(*args: Any, **kwargs: Any) -> Union[str, None]: + """Return a query string from supported args.""" + query: Union[ + str, + Mapping[str, QueryVariable], + Sequence[tuple[Union[str, istr], SimpleQuery]], + None, + ] + if kwargs: + if args: + msg = "Either kwargs or single query parameter must be present" + raise ValueError(msg) + query = kwargs + elif len(args) == 1: + query = args[0] + else: + raise ValueError("Either kwargs or single query parameter must be present") + + if query is None: + return None + if not query: + return "" + if type(query) is dict: + return get_str_query_from_sequence_iterable(query.items()) + if type(query) is str or isinstance(query, str): + return QUERY_QUOTER(query) + if isinstance(query, Mapping): + return get_str_query_from_sequence_iterable(query.items()) + if isinstance(query, (bytes, bytearray, memoryview)): + msg = "Invalid query type: bytes, bytearray and memoryview are forbidden" + raise TypeError(msg) + if isinstance(query, Sequence): + # We don't expect sequence values if we're given a list of pairs + # already; only mappings like builtin `dict` which can't have the + # same key pointing to multiple values are allowed to use + # `_query_seq_pairs`. + if TYPE_CHECKING: + query = cast(Sequence[tuple[Union[str, istr], SimpleQuery]], query) + return get_str_query_from_iterable(query) + raise TypeError( + "Invalid query type: only str, mapping or " + "sequence of (key, value) pairs is allowed" + ) diff --git a/lib/python3.12/site-packages/yarl/_quoters.py b/lib/python3.12/site-packages/yarl/_quoters.py new file mode 100644 index 0000000000000000000000000000000000000000..0feb5b141131697a6dc87df19941ffe6714b20ff --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_quoters.py @@ -0,0 +1,33 @@ +"""Quoting and unquoting utilities for URL parts.""" + +from typing import Union +from urllib.parse import quote + +from ._quoting import _Quoter, _Unquoter + +QUOTER = _Quoter(requote=False) +REQUOTER = _Quoter() +PATH_QUOTER = _Quoter(safe="@:", protected="/+", requote=False) +PATH_REQUOTER = _Quoter(safe="@:", protected="/+") +QUERY_QUOTER = _Quoter(safe="?/:@", protected="=+&;", qs=True, requote=False) +QUERY_REQUOTER = _Quoter(safe="?/:@", protected="=+&;", qs=True) +QUERY_PART_QUOTER = _Quoter(safe="?/:@", qs=True, requote=False) +FRAGMENT_QUOTER = _Quoter(safe="?/:@", requote=False) +FRAGMENT_REQUOTER = _Quoter(safe="?/:@") + +UNQUOTER = _Unquoter() +PATH_UNQUOTER = _Unquoter(unsafe="+") +PATH_SAFE_UNQUOTER = _Unquoter(ignore="/%", unsafe="+") +QS_UNQUOTER = _Unquoter(qs=True) +UNQUOTER_PLUS = _Unquoter(plus=True) # to match urllib.parse.unquote_plus + + +def human_quote(s: Union[str, None], unsafe: str) -> Union[str, None]: + if not s: + return s + for c in "%" + unsafe: + if c in s: + s = s.replace(c, f"%{ord(c):02X}") + if s.isprintable(): + return s + return "".join(c if c.isprintable() else quote(c) for c in s) diff --git a/lib/python3.12/site-packages/yarl/_quoting.py b/lib/python3.12/site-packages/yarl/_quoting.py new file mode 100644 index 0000000000000000000000000000000000000000..25d76c885cacaa815bb7e0149aedbe76c20f2228 --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_quoting.py @@ -0,0 +1,19 @@ +import os +import sys +from typing import TYPE_CHECKING + +__all__ = ("_Quoter", "_Unquoter") + + +NO_EXTENSIONS = bool(os.environ.get("YARL_NO_EXTENSIONS")) # type: bool +if sys.implementation.name != "cpython": + NO_EXTENSIONS = True + + +if TYPE_CHECKING or NO_EXTENSIONS: + from ._quoting_py import _Quoter, _Unquoter +else: + try: + from ._quoting_c import _Quoter, _Unquoter + except ImportError: # pragma: no cover + from ._quoting_py import _Quoter, _Unquoter # type: ignore[assignment] diff --git a/lib/python3.12/site-packages/yarl/_quoting_c.pyx b/lib/python3.12/site-packages/yarl/_quoting_c.pyx new file mode 100644 index 0000000000000000000000000000000000000000..dacf6b088c53e77a43f0435c06a2cbf51dcf4486 --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_quoting_c.pyx @@ -0,0 +1,451 @@ +from cpython.exc cimport PyErr_NoMemory +from cpython.mem cimport PyMem_Free, PyMem_Malloc, PyMem_Realloc +from cpython.unicode cimport ( + PyUnicode_DATA, + PyUnicode_DecodeASCII, + PyUnicode_DecodeUTF8Stateful, + PyUnicode_GET_LENGTH, + PyUnicode_KIND, + PyUnicode_READ, +) +from libc.stdint cimport uint8_t, uint64_t +from libc.string cimport memcpy, memset + +from string import ascii_letters, digits + + +cdef str GEN_DELIMS = ":/?#[]@" +cdef str SUB_DELIMS_WITHOUT_QS = "!$'()*," +cdef str SUB_DELIMS = SUB_DELIMS_WITHOUT_QS + '+?=;' +cdef str RESERVED = GEN_DELIMS + SUB_DELIMS +cdef str UNRESERVED = ascii_letters + digits + '-._~' +cdef str ALLOWED = UNRESERVED + SUB_DELIMS_WITHOUT_QS +cdef str QS = '+&=;' + +DEF BUF_SIZE = 8 * 1024 # 8KiB + +cdef inline Py_UCS4 _to_hex(uint8_t v) noexcept: + if v < 10: + return (v+0x30) # ord('0') == 0x30 + else: + return (v+0x41-10) # ord('A') == 0x41 + + +cdef inline int _from_hex(Py_UCS4 v) noexcept: + if '0' <= v <= '9': + return (v) - 0x30 # ord('0') == 0x30 + elif 'A' <= v <= 'F': + return (v) - 0x41 + 10 # ord('A') == 0x41 + elif 'a' <= v <= 'f': + return (v) - 0x61 + 10 # ord('a') == 0x61 + else: + return -1 + + +cdef inline int _is_lower_hex(Py_UCS4 v) noexcept: + return 'a' <= v <= 'f' + + +cdef inline long _restore_ch(Py_UCS4 d1, Py_UCS4 d2): + cdef int digit1 = _from_hex(d1) + if digit1 < 0: + return -1 + cdef int digit2 = _from_hex(d2) + if digit2 < 0: + return -1 + return digit1 << 4 | digit2 + + +cdef uint8_t ALLOWED_TABLE[16] +cdef uint8_t ALLOWED_NOTQS_TABLE[16] + + +cdef inline bint bit_at(uint8_t array[], uint64_t ch) noexcept: + return array[ch >> 3] & (1 << (ch & 7)) + + +cdef inline void set_bit(uint8_t array[], uint64_t ch) noexcept: + array[ch >> 3] |= (1 << (ch & 7)) + + +memset(ALLOWED_TABLE, 0, sizeof(ALLOWED_TABLE)) +memset(ALLOWED_NOTQS_TABLE, 0, sizeof(ALLOWED_NOTQS_TABLE)) + +for i in range(128): + if chr(i) in ALLOWED: + set_bit(ALLOWED_TABLE, i) + set_bit(ALLOWED_NOTQS_TABLE, i) + if chr(i) in QS: + set_bit(ALLOWED_NOTQS_TABLE, i) + +# ----------------- writer --------------------------- + +cdef struct Writer: + char *buf + bint heap_allocated_buf + Py_ssize_t size + Py_ssize_t pos + bint changed + + +cdef inline void _init_writer(Writer* writer, char* buf): + writer.buf = buf + writer.heap_allocated_buf = False + writer.size = BUF_SIZE + writer.pos = 0 + writer.changed = 0 + + +cdef inline void _release_writer(Writer* writer): + if writer.heap_allocated_buf: + PyMem_Free(writer.buf) + + +cdef inline int _write_char(Writer* writer, Py_UCS4 ch, bint changed): + cdef char * buf + cdef Py_ssize_t size + + if writer.pos == writer.size: + # reallocate + size = writer.size + BUF_SIZE + if not writer.heap_allocated_buf: + buf = PyMem_Malloc(size) + if buf == NULL: + PyErr_NoMemory() + return -1 + memcpy(buf, writer.buf, writer.size) + writer.heap_allocated_buf = True + else: + buf = PyMem_Realloc(writer.buf, size) + if buf == NULL: + PyErr_NoMemory() + return -1 + writer.buf = buf + writer.size = size + writer.buf[writer.pos] = ch + writer.pos += 1 + writer.changed |= changed + return 0 + + +cdef inline int _write_pct(Writer* writer, uint8_t ch, bint changed): + if _write_char(writer, '%', changed) < 0: + return -1 + if _write_char(writer, _to_hex(ch >> 4), changed) < 0: + return -1 + return _write_char(writer, _to_hex(ch & 0x0f), changed) + + +cdef inline int _write_utf8(Writer* writer, Py_UCS4 symbol): + cdef uint64_t utf = symbol + + if utf < 0x80: + return _write_pct(writer, utf, True) + elif utf < 0x800: + if _write_pct(writer, (0xc0 | (utf >> 6)), True) < 0: + return -1 + return _write_pct(writer, (0x80 | (utf & 0x3f)), True) + elif 0xD800 <= utf <= 0xDFFF: + # surogate pair, ignored + return 0 + elif utf < 0x10000: + if _write_pct(writer, (0xe0 | (utf >> 12)), True) < 0: + return -1 + if _write_pct(writer, (0x80 | ((utf >> 6) & 0x3f)), + True) < 0: + return -1 + return _write_pct(writer, (0x80 | (utf & 0x3f)), True) + elif utf > 0x10FFFF: + # symbol is too large + return 0 + else: + if _write_pct(writer, (0xf0 | (utf >> 18)), True) < 0: + return -1 + if _write_pct(writer, (0x80 | ((utf >> 12) & 0x3f)), + True) < 0: + return -1 + if _write_pct(writer, (0x80 | ((utf >> 6) & 0x3f)), + True) < 0: + return -1 + return _write_pct(writer, (0x80 | (utf & 0x3f)), True) + + +# --------------------- end writer -------------------------- + + +cdef class _Quoter: + cdef bint _qs + cdef bint _requote + + cdef uint8_t _safe_table[16] + cdef uint8_t _protected_table[16] + + def __init__( + self, *, str safe='', str protected='', bint qs=False, bint requote=True, + ): + cdef Py_UCS4 ch + + self._qs = qs + self._requote = requote + + if not self._qs: + memcpy(self._safe_table, + ALLOWED_NOTQS_TABLE, + sizeof(self._safe_table)) + else: + memcpy(self._safe_table, + ALLOWED_TABLE, + sizeof(self._safe_table)) + for ch in safe: + if ord(ch) > 127: + raise ValueError("Only safe symbols with ORD < 128 are allowed") + set_bit(self._safe_table, ch) + + memset(self._protected_table, 0, sizeof(self._protected_table)) + for ch in protected: + if ord(ch) > 127: + raise ValueError("Only safe symbols with ORD < 128 are allowed") + set_bit(self._safe_table, ch) + set_bit(self._protected_table, ch) + + def __call__(self, val): + if val is None: + return None + if type(val) is not str: + if isinstance(val, str): + # derived from str + val = str(val) + else: + raise TypeError("Argument should be str") + return self._do_quote_or_skip(val) + + cdef str _do_quote_or_skip(self, str val): + cdef char[BUF_SIZE] buffer + cdef Py_UCS4 ch + cdef Py_ssize_t length = PyUnicode_GET_LENGTH(val) + cdef Py_ssize_t idx = length + cdef bint must_quote = 0 + cdef Writer writer + cdef int kind = PyUnicode_KIND(val) + cdef const void *data = PyUnicode_DATA(val) + + # If everything in the string is in the safe + # table and all ASCII, we can skip quoting + while idx: + idx -= 1 + ch = PyUnicode_READ(kind, data, idx) + if ch >= 128 or not bit_at(self._safe_table, ch): + must_quote = 1 + break + + if not must_quote: + return val + + _init_writer(&writer, &buffer[0]) + try: + return self._do_quote(val, length, kind, data, &writer) + finally: + _release_writer(&writer) + + cdef str _do_quote( + self, + str val, + Py_ssize_t length, + int kind, + const void *data, + Writer *writer + ): + cdef Py_UCS4 ch + cdef long chl + cdef int changed + cdef Py_ssize_t idx = 0 + + while idx < length: + ch = PyUnicode_READ(kind, data, idx) + idx += 1 + if ch == '%' and self._requote and idx <= length - 2: + chl = _restore_ch( + PyUnicode_READ(kind, data, idx), + PyUnicode_READ(kind, data, idx + 1) + ) + if chl != -1: + ch = chl + idx += 2 + if ch < 128: + if bit_at(self._protected_table, ch): + if _write_pct(writer, ch, True) < 0: + raise + continue + + if bit_at(self._safe_table, ch): + if _write_char(writer, ch, True) < 0: + raise + continue + + changed = (_is_lower_hex(PyUnicode_READ(kind, data, idx - 2)) or + _is_lower_hex(PyUnicode_READ(kind, data, idx - 1))) + if _write_pct(writer, ch, changed) < 0: + raise + continue + else: + ch = '%' + + if self._write(writer, ch) < 0: + raise + + if not writer.changed: + return val + else: + return PyUnicode_DecodeASCII(writer.buf, writer.pos, "strict") + + cdef inline int _write(self, Writer *writer, Py_UCS4 ch): + if self._qs: + if ch == ' ': + return _write_char(writer, '+', True) + + if ch < 128 and bit_at(self._safe_table, ch): + return _write_char(writer, ch, False) + + return _write_utf8(writer, ch) + + +cdef class _Unquoter: + cdef str _ignore + cdef bint _has_ignore + cdef str _unsafe + cdef bytes _unsafe_bytes + cdef Py_ssize_t _unsafe_bytes_len + cdef const unsigned char * _unsafe_bytes_char + cdef bint _qs + cdef bint _plus # to match urllib.parse.unquote_plus + cdef _Quoter _quoter + cdef _Quoter _qs_quoter + + def __init__(self, *, ignore="", unsafe="", qs=False, plus=False): + self._ignore = ignore + self._has_ignore = bool(self._ignore) + self._unsafe = unsafe + # unsafe may only be extended ascii characters (0-255) + self._unsafe_bytes = self._unsafe.encode('ascii') + self._unsafe_bytes_len = len(self._unsafe_bytes) + self._unsafe_bytes_char = self._unsafe_bytes + self._qs = qs + self._plus = plus + self._quoter = _Quoter() + self._qs_quoter = _Quoter(qs=True) + + def __call__(self, val): + if val is None: + return None + if type(val) is not str: + if isinstance(val, str): + # derived from str + val = str(val) + else: + raise TypeError("Argument should be str") + return self._do_unquote(val) + + cdef str _do_unquote(self, str val): + cdef Py_ssize_t length = PyUnicode_GET_LENGTH(val) + if length == 0: + return val + + cdef list ret = [] + cdef char buffer[4] + cdef Py_ssize_t buflen = 0 + cdef Py_ssize_t consumed + cdef str unquoted + cdef Py_UCS4 ch = 0 + cdef long chl = 0 + cdef Py_ssize_t idx = 0 + cdef Py_ssize_t start_pct + cdef int kind = PyUnicode_KIND(val) + cdef const void *data = PyUnicode_DATA(val) + cdef bint changed = 0 + while idx < length: + ch = PyUnicode_READ(kind, data, idx) + idx += 1 + if ch == '%' and idx <= length - 2: + changed = 1 + chl = _restore_ch( + PyUnicode_READ(kind, data, idx), + PyUnicode_READ(kind, data, idx + 1) + ) + if chl != -1: + ch = chl + idx += 2 + assert buflen < 4 + buffer[buflen] = ch + buflen += 1 + try: + unquoted = PyUnicode_DecodeUTF8Stateful(buffer, buflen, + NULL, &consumed) + except UnicodeDecodeError: + start_pct = idx - buflen * 3 + buffer[0] = ch + buflen = 1 + ret.append(val[start_pct : idx - 3]) + try: + unquoted = PyUnicode_DecodeUTF8Stateful(buffer, buflen, + NULL, &consumed) + except UnicodeDecodeError: + buflen = 0 + ret.append(val[idx - 3 : idx]) + continue + if not unquoted: + assert consumed == 0 + continue + assert consumed == buflen + buflen = 0 + if self._qs and unquoted in '+=&;': + ret.append(self._qs_quoter(unquoted)) + elif ( + (self._unsafe_bytes_len and unquoted in self._unsafe) or + (self._has_ignore and unquoted in self._ignore) + ): + ret.append(self._quoter(unquoted)) + else: + ret.append(unquoted) + continue + else: + ch = '%' + + if buflen: + start_pct = idx - 1 - buflen * 3 + ret.append(val[start_pct : idx - 1]) + buflen = 0 + + if ch == '+': + if ( + (not self._qs and not self._plus) or + (self._unsafe_bytes_len and self._is_char_unsafe(ch)) + ): + ret.append('+') + else: + changed = 1 + ret.append(' ') + continue + + if self._unsafe_bytes_len and self._is_char_unsafe(ch): + changed = 1 + ret.append('%') + h = hex(ord(ch)).upper()[2:] + for ch in h: + ret.append(ch) + continue + + ret.append(ch) + + if not changed: + return val + + if buflen: + ret.append(val[length - buflen * 3 : length]) + + return ''.join(ret) + + cdef inline bint _is_char_unsafe(self, Py_UCS4 ch): + for i in range(self._unsafe_bytes_len): + if ch == self._unsafe_bytes_char[i]: + return True + return False diff --git a/lib/python3.12/site-packages/yarl/_quoting_py.py b/lib/python3.12/site-packages/yarl/_quoting_py.py new file mode 100644 index 0000000000000000000000000000000000000000..80bf07febf30d2ab3702c08c635ab83c20db877a --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_quoting_py.py @@ -0,0 +1,213 @@ +import codecs +import re +from string import ascii_letters, ascii_lowercase, digits +from typing import Union, overload + +BASCII_LOWERCASE = ascii_lowercase.encode("ascii") +BPCT_ALLOWED = {f"%{i:02X}".encode("ascii") for i in range(256)} +GEN_DELIMS = ":/?#[]@" +SUB_DELIMS_WITHOUT_QS = "!$'()*," +SUB_DELIMS = SUB_DELIMS_WITHOUT_QS + "+&=;" +RESERVED = GEN_DELIMS + SUB_DELIMS +UNRESERVED = ascii_letters + digits + "-._~" +ALLOWED = UNRESERVED + SUB_DELIMS_WITHOUT_QS + + +_IS_HEX = re.compile(b"[A-Z0-9][A-Z0-9]") +_IS_HEX_STR = re.compile("[A-Fa-f0-9][A-Fa-f0-9]") + +utf8_decoder = codecs.getincrementaldecoder("utf-8") + + +class _Quoter: + def __init__( + self, + *, + safe: str = "", + protected: str = "", + qs: bool = False, + requote: bool = True, + ) -> None: + self._safe = safe + self._protected = protected + self._qs = qs + self._requote = requote + + @overload + def __call__(self, val: str) -> str: ... + @overload + def __call__(self, val: None) -> None: ... + def __call__(self, val: Union[str, None]) -> Union[str, None]: + if val is None: + return None + if not isinstance(val, str): + raise TypeError("Argument should be str") + if not val: + return "" + bval = val.encode("utf8", errors="ignore") + ret = bytearray() + pct = bytearray() + safe = self._safe + safe += ALLOWED + if not self._qs: + safe += "+&=;" + safe += self._protected + bsafe = safe.encode("ascii") + idx = 0 + while idx < len(bval): + ch = bval[idx] + idx += 1 + + if pct: + if ch in BASCII_LOWERCASE: + ch = ch - 32 # convert to uppercase + pct.append(ch) + if len(pct) == 3: # pragma: no branch # peephole optimizer + buf = pct[1:] + if not _IS_HEX.match(buf): + ret.extend(b"%25") + pct.clear() + idx -= 2 + continue + try: + unquoted = chr(int(pct[1:].decode("ascii"), base=16)) + except ValueError: + ret.extend(b"%25") + pct.clear() + idx -= 2 + continue + + if unquoted in self._protected: + ret.extend(pct) + elif unquoted in safe: + ret.append(ord(unquoted)) + else: + ret.extend(pct) + pct.clear() + + # special case, if we have only one char after "%" + elif len(pct) == 2 and idx == len(bval): + ret.extend(b"%25") + pct.clear() + idx -= 1 + + continue + + elif ch == ord("%") and self._requote: + pct.clear() + pct.append(ch) + + # special case if "%" is last char + if idx == len(bval): + ret.extend(b"%25") + + continue + + if self._qs and ch == ord(" "): + ret.append(ord("+")) + continue + if ch in bsafe: + ret.append(ch) + continue + + ret.extend((f"%{ch:02X}").encode("ascii")) + + ret2 = ret.decode("ascii") + if ret2 == val: + return val + return ret2 + + +class _Unquoter: + def __init__( + self, + *, + ignore: str = "", + unsafe: str = "", + qs: bool = False, + plus: bool = False, + ) -> None: + self._ignore = ignore + self._unsafe = unsafe + self._qs = qs + self._plus = plus # to match urllib.parse.unquote_plus + self._quoter = _Quoter() + self._qs_quoter = _Quoter(qs=True) + + @overload + def __call__(self, val: str) -> str: ... + @overload + def __call__(self, val: None) -> None: ... + def __call__(self, val: Union[str, None]) -> Union[str, None]: + if val is None: + return None + if not isinstance(val, str): + raise TypeError("Argument should be str") + if not val: + return "" + decoder = utf8_decoder() + ret = [] + idx = 0 + while idx < len(val): + ch = val[idx] + idx += 1 + if ch == "%" and idx <= len(val) - 2: + pct = val[idx : idx + 2] + if _IS_HEX_STR.fullmatch(pct): + b = bytes([int(pct, base=16)]) + idx += 2 + try: + unquoted = decoder.decode(b) + except UnicodeDecodeError: + start_pct = idx - 3 - len(decoder.buffer) * 3 + ret.append(val[start_pct : idx - 3]) + decoder.reset() + try: + unquoted = decoder.decode(b) + except UnicodeDecodeError: + ret.append(val[idx - 3 : idx]) + continue + if not unquoted: + continue + if self._qs and unquoted in "+=&;": + to_add = self._qs_quoter(unquoted) + if to_add is None: # pragma: no cover + raise RuntimeError("Cannot quote None") + ret.append(to_add) + elif unquoted in self._unsafe or unquoted in self._ignore: + to_add = self._quoter(unquoted) + if to_add is None: # pragma: no cover + raise RuntimeError("Cannot quote None") + ret.append(to_add) + else: + ret.append(unquoted) + continue + + if decoder.buffer: + start_pct = idx - 1 - len(decoder.buffer) * 3 + ret.append(val[start_pct : idx - 1]) + decoder.reset() + + if ch == "+": + if (not self._qs and not self._plus) or ch in self._unsafe: + ret.append("+") + else: + ret.append(" ") + continue + + if ch in self._unsafe: + ret.append("%") + h = hex(ord(ch)).upper()[2:] + for ch in h: + ret.append(ch) + continue + + ret.append(ch) + + if decoder.buffer: + ret.append(val[-len(decoder.buffer) * 3 :]) + + ret2 = "".join(ret) + if ret2 == val: + return val + return ret2 diff --git a/lib/python3.12/site-packages/yarl/_url.py b/lib/python3.12/site-packages/yarl/_url.py new file mode 100644 index 0000000000000000000000000000000000000000..527a576ee68f8fbeff941ef67bb1590f940862d0 --- /dev/null +++ b/lib/python3.12/site-packages/yarl/_url.py @@ -0,0 +1,1622 @@ +import re +import sys +import warnings +from collections.abc import Mapping, Sequence +from enum import Enum +from functools import _CacheInfo, lru_cache +from ipaddress import ip_address +from typing import ( + TYPE_CHECKING, + Any, + NoReturn, + TypedDict, + TypeVar, + Union, + cast, + overload, +) +from urllib.parse import SplitResult, uses_relative + +import idna +from multidict import MultiDict, MultiDictProxy, istr +from propcache.api import under_cached_property as cached_property + +from ._parse import ( + USES_AUTHORITY, + SplitURLType, + make_netloc, + query_to_pairs, + split_netloc, + split_url, + unsplit_result, +) +from ._path import normalize_path, normalize_path_segments +from ._query import ( + Query, + QueryVariable, + SimpleQuery, + get_str_query, + get_str_query_from_iterable, + get_str_query_from_sequence_iterable, +) +from ._quoters import ( + FRAGMENT_QUOTER, + FRAGMENT_REQUOTER, + PATH_QUOTER, + PATH_REQUOTER, + PATH_SAFE_UNQUOTER, + PATH_UNQUOTER, + QS_UNQUOTER, + QUERY_QUOTER, + QUERY_REQUOTER, + QUOTER, + REQUOTER, + UNQUOTER, + human_quote, +) + +DEFAULT_PORTS = {"http": 80, "https": 443, "ws": 80, "wss": 443, "ftp": 21} +USES_RELATIVE = frozenset(uses_relative) + +# Special schemes https://url.spec.whatwg.org/#special-scheme +# are not allowed to have an empty host https://url.spec.whatwg.org/#url-representation +SCHEME_REQUIRES_HOST = frozenset(("http", "https", "ws", "wss", "ftp")) + + +# reg-name: unreserved / pct-encoded / sub-delims +# this pattern matches anything that is *not* in those classes. and is only used +# on lower-cased ASCII values. +NOT_REG_NAME = re.compile( + r""" + # any character not in the unreserved or sub-delims sets, plus % + # (validated with the additional check for pct-encoded sequences below) + [^a-z0-9\-._~!$&'()*+,;=%] + | + # % only allowed if it is part of a pct-encoded + # sequence of 2 hex digits. + %(?![0-9a-f]{2}) + """, + re.VERBOSE, +) + +_T = TypeVar("_T") + +if sys.version_info >= (3, 11): + from typing import Self +else: + Self = Any + + +class UndefinedType(Enum): + """Singleton type for use with not set sentinel values.""" + + _singleton = 0 + + +UNDEFINED = UndefinedType._singleton + + +class CacheInfo(TypedDict): + """Host encoding cache.""" + + idna_encode: _CacheInfo + idna_decode: _CacheInfo + ip_address: _CacheInfo + host_validate: _CacheInfo + encode_host: _CacheInfo + + +class _InternalURLCache(TypedDict, total=False): + _val: SplitURLType + _origin: "URL" + absolute: bool + hash: int + scheme: str + raw_authority: str + authority: str + raw_user: Union[str, None] + user: Union[str, None] + raw_password: Union[str, None] + password: Union[str, None] + raw_host: Union[str, None] + host: Union[str, None] + host_subcomponent: Union[str, None] + host_port_subcomponent: Union[str, None] + port: Union[int, None] + explicit_port: Union[int, None] + raw_path: str + path: str + _parsed_query: list[tuple[str, str]] + query: "MultiDictProxy[str]" + raw_query_string: str + query_string: str + path_qs: str + raw_path_qs: str + raw_fragment: str + fragment: str + raw_parts: tuple[str, ...] + parts: tuple[str, ...] + parent: "URL" + raw_name: str + name: str + raw_suffix: str + suffix: str + raw_suffixes: tuple[str, ...] + suffixes: tuple[str, ...] + + +def rewrite_module(obj: _T) -> _T: + obj.__module__ = "yarl" + return obj + + +@lru_cache +def encode_url(url_str: str) -> "URL": + """Parse unencoded URL.""" + cache: _InternalURLCache = {} + host: Union[str, None] + scheme, netloc, path, query, fragment = split_url(url_str) + if not netloc: # netloc + host = "" + else: + if ":" in netloc or "@" in netloc or "[" in netloc: + # Complex netloc + username, password, host, port = split_netloc(netloc) + else: + username = password = port = None + host = netloc + if host is None: + if scheme in SCHEME_REQUIRES_HOST: + msg = ( + "Invalid URL: host is required for " + f"absolute urls with the {scheme} scheme" + ) + raise ValueError(msg) + else: + host = "" + host = _encode_host(host, validate_host=False) + # Remove brackets as host encoder adds back brackets for IPv6 addresses + cache["raw_host"] = host[1:-1] if "[" in host else host + cache["explicit_port"] = port + if password is None and username is None: + # Fast path for URLs without user, password + netloc = host if port is None else f"{host}:{port}" + cache["raw_user"] = None + cache["raw_password"] = None + else: + raw_user = REQUOTER(username) if username else username + raw_password = REQUOTER(password) if password else password + netloc = make_netloc(raw_user, raw_password, host, port) + cache["raw_user"] = raw_user + cache["raw_password"] = raw_password + + if path: + path = PATH_REQUOTER(path) + if netloc and "." in path: + path = normalize_path(path) + if query: + query = QUERY_REQUOTER(query) + if fragment: + fragment = FRAGMENT_REQUOTER(fragment) + + cache["scheme"] = scheme + cache["raw_path"] = "/" if not path and netloc else path + cache["raw_query_string"] = query + cache["raw_fragment"] = fragment + + self = object.__new__(URL) + self._scheme = scheme + self._netloc = netloc + self._path = path + self._query = query + self._fragment = fragment + self._cache = cache + return self + + +@lru_cache +def pre_encoded_url(url_str: str) -> "URL": + """Parse pre-encoded URL.""" + self = object.__new__(URL) + val = split_url(url_str) + self._scheme, self._netloc, self._path, self._query, self._fragment = val + self._cache = {} + return self + + +@lru_cache +def build_pre_encoded_url( + scheme: str, + authority: str, + user: Union[str, None], + password: Union[str, None], + host: str, + port: Union[int, None], + path: str, + query_string: str, + fragment: str, +) -> "URL": + """Build a pre-encoded URL from parts.""" + self = object.__new__(URL) + self._scheme = scheme + if authority: + self._netloc = authority + elif host: + if port is not None: + port = None if port == DEFAULT_PORTS.get(scheme) else port + if user is None and password is None: + self._netloc = host if port is None else f"{host}:{port}" + else: + self._netloc = make_netloc(user, password, host, port) + else: + self._netloc = "" + self._path = path + self._query = query_string + self._fragment = fragment + self._cache = {} + return self + + +def from_parts_uncached( + scheme: str, netloc: str, path: str, query: str, fragment: str +) -> "URL": + """Create a new URL from parts.""" + self = object.__new__(URL) + self._scheme = scheme + self._netloc = netloc + self._path = path + self._query = query + self._fragment = fragment + self._cache = {} + return self + + +from_parts = lru_cache(from_parts_uncached) + + +@rewrite_module +class URL: + # Don't derive from str + # follow pathlib.Path design + # probably URL will not suffer from pathlib problems: + # it's intended for libraries like aiohttp, + # not to be passed into standard library functions like os.open etc. + + # URL grammar (RFC 3986) + # pct-encoded = "%" HEXDIG HEXDIG + # reserved = gen-delims / sub-delims + # gen-delims = ":" / "/" / "?" / "#" / "[" / "]" / "@" + # sub-delims = "!" / "$" / "&" / "'" / "(" / ")" + # / "*" / "+" / "," / ";" / "=" + # unreserved = ALPHA / DIGIT / "-" / "." / "_" / "~" + # URI = scheme ":" hier-part [ "?" query ] [ "#" fragment ] + # hier-part = "//" authority path-abempty + # / path-absolute + # / path-rootless + # / path-empty + # scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." ) + # authority = [ userinfo "@" ] host [ ":" port ] + # userinfo = *( unreserved / pct-encoded / sub-delims / ":" ) + # host = IP-literal / IPv4address / reg-name + # IP-literal = "[" ( IPv6address / IPvFuture ) "]" + # IPvFuture = "v" 1*HEXDIG "." 1*( unreserved / sub-delims / ":" ) + # IPv6address = 6( h16 ":" ) ls32 + # / "::" 5( h16 ":" ) ls32 + # / [ h16 ] "::" 4( h16 ":" ) ls32 + # / [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + # / [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + # / [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + # / [ *4( h16 ":" ) h16 ] "::" ls32 + # / [ *5( h16 ":" ) h16 ] "::" h16 + # / [ *6( h16 ":" ) h16 ] "::" + # ls32 = ( h16 ":" h16 ) / IPv4address + # ; least-significant 32 bits of address + # h16 = 1*4HEXDIG + # ; 16 bits of address represented in hexadecimal + # IPv4address = dec-octet "." dec-octet "." dec-octet "." dec-octet + # dec-octet = DIGIT ; 0-9 + # / %x31-39 DIGIT ; 10-99 + # / "1" 2DIGIT ; 100-199 + # / "2" %x30-34 DIGIT ; 200-249 + # / "25" %x30-35 ; 250-255 + # reg-name = *( unreserved / pct-encoded / sub-delims ) + # port = *DIGIT + # path = path-abempty ; begins with "/" or is empty + # / path-absolute ; begins with "/" but not "//" + # / path-noscheme ; begins with a non-colon segment + # / path-rootless ; begins with a segment + # / path-empty ; zero characters + # path-abempty = *( "/" segment ) + # path-absolute = "/" [ segment-nz *( "/" segment ) ] + # path-noscheme = segment-nz-nc *( "/" segment ) + # path-rootless = segment-nz *( "/" segment ) + # path-empty = 0 + # segment = *pchar + # segment-nz = 1*pchar + # segment-nz-nc = 1*( unreserved / pct-encoded / sub-delims / "@" ) + # ; non-zero-length segment without any colon ":" + # pchar = unreserved / pct-encoded / sub-delims / ":" / "@" + # query = *( pchar / "/" / "?" ) + # fragment = *( pchar / "/" / "?" ) + # URI-reference = URI / relative-ref + # relative-ref = relative-part [ "?" query ] [ "#" fragment ] + # relative-part = "//" authority path-abempty + # / path-absolute + # / path-noscheme + # / path-empty + # absolute-URI = scheme ":" hier-part [ "?" query ] + __slots__ = ("_cache", "_scheme", "_netloc", "_path", "_query", "_fragment") + + _cache: _InternalURLCache + _scheme: str + _netloc: str + _path: str + _query: str + _fragment: str + + def __new__( + cls, + val: Union[str, SplitResult, "URL", UndefinedType] = UNDEFINED, + *, + encoded: bool = False, + strict: Union[bool, None] = None, + ) -> "URL": + if strict is not None: # pragma: no cover + warnings.warn("strict parameter is ignored") + if type(val) is str: + return pre_encoded_url(val) if encoded else encode_url(val) + if type(val) is cls: + return val + if type(val) is SplitResult: + if not encoded: + raise ValueError("Cannot apply decoding to SplitResult") + return from_parts(*val) + if isinstance(val, str): + return pre_encoded_url(str(val)) if encoded else encode_url(str(val)) + if val is UNDEFINED: + # Special case for UNDEFINED since it might be unpickling and we do + # not want to cache as the `__set_state__` call would mutate the URL + # object in the `pre_encoded_url` or `encoded_url` caches. + self = object.__new__(URL) + self._scheme = self._netloc = self._path = self._query = self._fragment = "" + self._cache = {} + return self + raise TypeError("Constructor parameter should be str") + + @classmethod + def build( + cls, + *, + scheme: str = "", + authority: str = "", + user: Union[str, None] = None, + password: Union[str, None] = None, + host: str = "", + port: Union[int, None] = None, + path: str = "", + query: Union[Query, None] = None, + query_string: str = "", + fragment: str = "", + encoded: bool = False, + ) -> "URL": + """Creates and returns a new URL""" + + if authority and (user or password or host or port): + raise ValueError( + 'Can\'t mix "authority" with "user", "password", "host" or "port".' + ) + if port is not None and not isinstance(port, int): + raise TypeError(f"The port is required to be int, got {type(port)!r}.") + if port and not host: + raise ValueError('Can\'t build URL with "port" but without "host".') + if query and query_string: + raise ValueError('Only one of "query" or "query_string" should be passed') + if ( + scheme is None # type: ignore[redundant-expr] + or authority is None # type: ignore[redundant-expr] + or host is None # type: ignore[redundant-expr] + or path is None # type: ignore[redundant-expr] + or query_string is None # type: ignore[redundant-expr] + or fragment is None + ): + raise TypeError( + 'NoneType is illegal for "scheme", "authority", "host", "path", ' + '"query_string", and "fragment" args, use empty string instead.' + ) + + if query: + query_string = get_str_query(query) or "" + + if encoded: + return build_pre_encoded_url( + scheme, + authority, + user, + password, + host, + port, + path, + query_string, + fragment, + ) + + self = object.__new__(URL) + self._scheme = scheme + _host: Union[str, None] = None + if authority: + user, password, _host, port = split_netloc(authority) + _host = _encode_host(_host, validate_host=False) if _host else "" + elif host: + _host = _encode_host(host, validate_host=True) + else: + self._netloc = "" + + if _host is not None: + if port is not None: + port = None if port == DEFAULT_PORTS.get(scheme) else port + if user is None and password is None: + self._netloc = _host if port is None else f"{_host}:{port}" + else: + self._netloc = make_netloc(user, password, _host, port, True) + + path = PATH_QUOTER(path) if path else path + if path and self._netloc: + if "." in path: + path = normalize_path(path) + if path[0] != "/": + msg = ( + "Path in a URL with authority should " + "start with a slash ('/') if set" + ) + raise ValueError(msg) + + self._path = path + if not query and query_string: + query_string = QUERY_QUOTER(query_string) + self._query = query_string + self._fragment = FRAGMENT_QUOTER(fragment) if fragment else fragment + self._cache = {} + return self + + def __init_subclass__(cls) -> NoReturn: + raise TypeError(f"Inheriting a class {cls!r} from URL is forbidden") + + def __str__(self) -> str: + if not self._path and self._netloc and (self._query or self._fragment): + path = "/" + else: + path = self._path + if (port := self.explicit_port) is not None and port == DEFAULT_PORTS.get( + self._scheme + ): + # port normalization - using None for default ports to remove from rendering + # https://datatracker.ietf.org/doc/html/rfc3986.html#section-6.2.3 + host = self.host_subcomponent + netloc = make_netloc(self.raw_user, self.raw_password, host, None) + else: + netloc = self._netloc + return unsplit_result(self._scheme, netloc, path, self._query, self._fragment) + + def __repr__(self) -> str: + return f"{self.__class__.__name__}('{str(self)}')" + + def __bytes__(self) -> bytes: + return str(self).encode("ascii") + + def __eq__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + + path1 = "/" if not self._path and self._netloc else self._path + path2 = "/" if not other._path and other._netloc else other._path + return ( + self._scheme == other._scheme + and self._netloc == other._netloc + and path1 == path2 + and self._query == other._query + and self._fragment == other._fragment + ) + + def __hash__(self) -> int: + if (ret := self._cache.get("hash")) is None: + path = "/" if not self._path and self._netloc else self._path + ret = self._cache["hash"] = hash( + (self._scheme, self._netloc, path, self._query, self._fragment) + ) + return ret + + def __le__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val <= other._val + + def __lt__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val < other._val + + def __ge__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val >= other._val + + def __gt__(self, other: object) -> bool: + if type(other) is not URL: + return NotImplemented + return self._val > other._val + + def __truediv__(self, name: str) -> "URL": + if not isinstance(name, str): + return NotImplemented # type: ignore[unreachable] + return self._make_child((str(name),)) + + def __mod__(self, query: Query) -> "URL": + return self.update_query(query) + + def __bool__(self) -> bool: + return bool(self._netloc or self._path or self._query or self._fragment) + + def __getstate__(self) -> tuple[SplitResult]: + return (tuple.__new__(SplitResult, self._val),) + + def __setstate__( + self, state: Union[tuple[SplitURLType], tuple[None, _InternalURLCache]] + ) -> None: + if state[0] is None and isinstance(state[1], dict): + # default style pickle + val = state[1]["_val"] + else: + unused: list[object] + val, *unused = state + self._scheme, self._netloc, self._path, self._query, self._fragment = val + self._cache = {} + + def _cache_netloc(self) -> None: + """Cache the netloc parts of the URL.""" + c = self._cache + split_loc = split_netloc(self._netloc) + c["raw_user"], c["raw_password"], c["raw_host"], c["explicit_port"] = split_loc + + def is_absolute(self) -> bool: + """A check for absolute URLs. + + Return True for absolute ones (having scheme or starting + with //), False otherwise. + + Is is preferred to call the .absolute property instead + as it is cached. + """ + return self.absolute + + def is_default_port(self) -> bool: + """A check for default port. + + Return True if port is default for specified scheme, + e.g. 'http://python.org' or 'http://python.org:80', False + otherwise. + + Return False for relative URLs. + + """ + if (explicit := self.explicit_port) is None: + # If the explicit port is None, then the URL must be + # using the default port unless its a relative URL + # which does not have an implicit port / default port + return self._netloc != "" + return explicit == DEFAULT_PORTS.get(self._scheme) + + def origin(self) -> "URL": + """Return an URL with scheme, host and port parts only. + + user, password, path, query and fragment are removed. + + """ + # TODO: add a keyword-only option for keeping user/pass maybe? + return self._origin + + @cached_property + def _val(self) -> SplitURLType: + return (self._scheme, self._netloc, self._path, self._query, self._fragment) + + @cached_property + def _origin(self) -> "URL": + """Return an URL with scheme, host and port parts only. + + user, password, path, query and fragment are removed. + """ + if not (netloc := self._netloc): + raise ValueError("URL should be absolute") + if not (scheme := self._scheme): + raise ValueError("URL should have scheme") + if "@" in netloc: + encoded_host = self.host_subcomponent + netloc = make_netloc(None, None, encoded_host, self.explicit_port) + elif not self._path and not self._query and not self._fragment: + return self + return from_parts(scheme, netloc, "", "", "") + + def relative(self) -> "URL": + """Return a relative part of the URL. + + scheme, user, password, host and port are removed. + + """ + if not self._netloc: + raise ValueError("URL should be absolute") + return from_parts("", "", self._path, self._query, self._fragment) + + @cached_property + def absolute(self) -> bool: + """A check for absolute URLs. + + Return True for absolute ones (having scheme or starting + with //), False otherwise. + + """ + # `netloc`` is an empty string for relative URLs + # Checking `netloc` is faster than checking `hostname` + # because `hostname` is a property that does some extra work + # to parse the host from the `netloc` + return self._netloc != "" + + @cached_property + def scheme(self) -> str: + """Scheme for absolute URLs. + + Empty string for relative URLs or URLs starting with // + + """ + return self._scheme + + @cached_property + def raw_authority(self) -> str: + """Encoded authority part of URL. + + Empty string for relative URLs. + + """ + return self._netloc + + @cached_property + def authority(self) -> str: + """Decoded authority part of URL. + + Empty string for relative URLs. + + """ + return make_netloc(self.user, self.password, self.host, self.port) + + @cached_property + def raw_user(self) -> Union[str, None]: + """Encoded user part of URL. + + None if user is missing. + + """ + # not .username + self._cache_netloc() + return self._cache["raw_user"] + + @cached_property + def user(self) -> Union[str, None]: + """Decoded user part of URL. + + None if user is missing. + + """ + if (raw_user := self.raw_user) is None: + return None + return UNQUOTER(raw_user) + + @cached_property + def raw_password(self) -> Union[str, None]: + """Encoded password part of URL. + + None if password is missing. + + """ + self._cache_netloc() + return self._cache["raw_password"] + + @cached_property + def password(self) -> Union[str, None]: + """Decoded password part of URL. + + None if password is missing. + + """ + if (raw_password := self.raw_password) is None: + return None + return UNQUOTER(raw_password) + + @cached_property + def raw_host(self) -> Union[str, None]: + """Encoded host part of URL. + + None for relative URLs. + + When working with IPv6 addresses, use the `host_subcomponent` property instead + as it will return the host subcomponent with brackets. + """ + # Use host instead of hostname for sake of shortness + # May add .hostname prop later + self._cache_netloc() + return self._cache["raw_host"] + + @cached_property + def host(self) -> Union[str, None]: + """Decoded host part of URL. + + None for relative URLs. + + """ + if (raw := self.raw_host) is None: + return None + if raw and raw[-1].isdigit() or ":" in raw: + # IP addresses are never IDNA encoded + return raw + return _idna_decode(raw) + + @cached_property + def host_subcomponent(self) -> Union[str, None]: + """Return the host subcomponent part of URL. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + + Examples: + - `http://example.com:8080` -> `example.com` + - `http://example.com:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1` + - `https://[::1]:8443` -> `[::1]` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + return f"[{raw}]" if ":" in raw else raw + + @cached_property + def host_port_subcomponent(self) -> Union[str, None]: + """Return the host and port subcomponent part of URL. + + Trailing dots are removed from the host part. + + This value is suitable for use in the Host header of an HTTP request. + + None for relative URLs. + + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + `IP-literal = "[" ( IPv6address / IPvFuture ) "]"` + https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.3 + port = *DIGIT + + Examples: + - `http://example.com:8080` -> `example.com:8080` + - `http://example.com:80` -> `example.com` + - `http://example.com.:80` -> `example.com` + - `https://127.0.0.1:8443` -> `127.0.0.1:8443` + - `https://[::1]:8443` -> `[::1]:8443` + - `http://[::1]` -> `[::1]` + + """ + if (raw := self.raw_host) is None: + return None + if raw[-1] == ".": + # Remove all trailing dots from the netloc as while + # they are valid FQDNs in DNS, TLS validation fails. + # See https://github.com/aio-libs/aiohttp/issues/3636. + # To avoid string manipulation we only call rstrip if + # the last character is a dot. + raw = raw.rstrip(".") + port = self.explicit_port + if port is None or port == DEFAULT_PORTS.get(self._scheme): + return f"[{raw}]" if ":" in raw else raw + return f"[{raw}]:{port}" if ":" in raw else f"{raw}:{port}" + + @cached_property + def port(self) -> Union[int, None]: + """Port part of URL, with scheme-based fallback. + + None for relative URLs or URLs without explicit port and + scheme without default port substitution. + + """ + if (explicit_port := self.explicit_port) is not None: + return explicit_port + return DEFAULT_PORTS.get(self._scheme) + + @cached_property + def explicit_port(self) -> Union[int, None]: + """Port part of URL, without scheme-based fallback. + + None for relative URLs or URLs without explicit port. + + """ + self._cache_netloc() + return self._cache["explicit_port"] + + @cached_property + def raw_path(self) -> str: + """Encoded path of URL. + + / for absolute URLs without path part. + + """ + return self._path if self._path or not self._netloc else "/" + + @cached_property + def path(self) -> str: + """Decoded path of URL. + + / for absolute URLs without path part. + + """ + return PATH_UNQUOTER(self._path) if self._path else "/" if self._netloc else "" + + @cached_property + def path_safe(self) -> str: + """Decoded path of URL. + + / for absolute URLs without path part. + + / (%2F) and % (%25) are not decoded + + """ + if self._path: + return PATH_SAFE_UNQUOTER(self._path) + return "/" if self._netloc else "" + + @cached_property + def _parsed_query(self) -> list[tuple[str, str]]: + """Parse query part of URL.""" + return query_to_pairs(self._query) + + @cached_property + def query(self) -> "MultiDictProxy[str]": + """A MultiDictProxy representing parsed query parameters in decoded + representation. + + Empty value if URL has no query part. + + """ + return MultiDictProxy(MultiDict(self._parsed_query)) + + @cached_property + def raw_query_string(self) -> str: + """Encoded query part of URL. + + Empty string if query is missing. + + """ + return self._query + + @cached_property + def query_string(self) -> str: + """Decoded query part of URL. + + Empty string if query is missing. + + """ + return QS_UNQUOTER(self._query) if self._query else "" + + @cached_property + def path_qs(self) -> str: + """Decoded path of URL with query.""" + return self.path if not (q := self.query_string) else f"{self.path}?{q}" + + @cached_property + def raw_path_qs(self) -> str: + """Encoded path of URL with query.""" + if q := self._query: + return f"{self._path}?{q}" if self._path or not self._netloc else f"/?{q}" + return self._path if self._path or not self._netloc else "/" + + @cached_property + def raw_fragment(self) -> str: + """Encoded fragment part of URL. + + Empty string if fragment is missing. + + """ + return self._fragment + + @cached_property + def fragment(self) -> str: + """Decoded fragment part of URL. + + Empty string if fragment is missing. + + """ + return UNQUOTER(self._fragment) if self._fragment else "" + + @cached_property + def raw_parts(self) -> tuple[str, ...]: + """A tuple containing encoded *path* parts. + + ('/',) for absolute URLs if *path* is missing. + + """ + path = self._path + if self._netloc: + return ("/", *path[1:].split("/")) if path else ("/",) + if path and path[0] == "/": + return ("/", *path[1:].split("/")) + return tuple(path.split("/")) + + @cached_property + def parts(self) -> tuple[str, ...]: + """A tuple containing decoded *path* parts. + + ('/',) for absolute URLs if *path* is missing. + + """ + return tuple(UNQUOTER(part) for part in self.raw_parts) + + @cached_property + def parent(self) -> "URL": + """A new URL with last part of path removed and cleaned up query and + fragment. + + """ + path = self._path + if not path or path == "/": + if self._fragment or self._query: + return from_parts(self._scheme, self._netloc, path, "", "") + return self + parts = path.split("/") + return from_parts(self._scheme, self._netloc, "/".join(parts[:-1]), "", "") + + @cached_property + def raw_name(self) -> str: + """The last part of raw_parts.""" + parts = self.raw_parts + if not self._netloc: + return parts[-1] + parts = parts[1:] + return parts[-1] if parts else "" + + @cached_property + def name(self) -> str: + """The last part of parts.""" + return UNQUOTER(self.raw_name) + + @cached_property + def raw_suffix(self) -> str: + name = self.raw_name + i = name.rfind(".") + return name[i:] if 0 < i < len(name) - 1 else "" + + @cached_property + def suffix(self) -> str: + return UNQUOTER(self.raw_suffix) + + @cached_property + def raw_suffixes(self) -> tuple[str, ...]: + name = self.raw_name + if name.endswith("."): + return () + name = name.lstrip(".") + return tuple("." + suffix for suffix in name.split(".")[1:]) + + @cached_property + def suffixes(self) -> tuple[str, ...]: + return tuple(UNQUOTER(suffix) for suffix in self.raw_suffixes) + + def _make_child(self, paths: "Sequence[str]", encoded: bool = False) -> "URL": + """ + add paths to self._path, accounting for absolute vs relative paths, + keep existing, but do not create new, empty segments + """ + parsed: list[str] = [] + needs_normalize: bool = False + for idx, path in enumerate(reversed(paths)): + # empty segment of last is not removed + last = idx == 0 + if path and path[0] == "/": + raise ValueError( + f"Appending path {path!r} starting from slash is forbidden" + ) + # We need to quote the path if it is not already encoded + # This cannot be done at the end because the existing + # path is already quoted and we do not want to double quote + # the existing path. + path = path if encoded else PATH_QUOTER(path) + needs_normalize |= "." in path + segments = path.split("/") + segments.reverse() + # remove trailing empty segment for all but the last path + parsed += segments[1:] if not last and segments[0] == "" else segments + + if (path := self._path) and (old_segments := path.split("/")): + # If the old path ends with a slash, the last segment is an empty string + # and should be removed before adding the new path segments. + old = old_segments[:-1] if old_segments[-1] == "" else old_segments + old.reverse() + parsed += old + + # If the netloc is present, inject a leading slash when adding a + # path to an absolute URL where there was none before. + if (netloc := self._netloc) and parsed and parsed[-1] != "": + parsed.append("") + + parsed.reverse() + if not netloc or not needs_normalize: + return from_parts(self._scheme, netloc, "/".join(parsed), "", "") + + path = "/".join(normalize_path_segments(parsed)) + # If normalizing the path segments removed the leading slash, add it back. + if path and path[0] != "/": + path = f"/{path}" + return from_parts(self._scheme, netloc, path, "", "") + + def with_scheme(self, scheme: str) -> "URL": + """Return a new URL with scheme replaced.""" + # N.B. doesn't cleanup query/fragment + if not isinstance(scheme, str): + raise TypeError("Invalid scheme type") + lower_scheme = scheme.lower() + netloc = self._netloc + if not netloc and lower_scheme in SCHEME_REQUIRES_HOST: + msg = ( + "scheme replacement is not allowed for " + f"relative URLs for the {lower_scheme} scheme" + ) + raise ValueError(msg) + return from_parts(lower_scheme, netloc, self._path, self._query, self._fragment) + + def with_user(self, user: Union[str, None]) -> "URL": + """Return a new URL with user replaced. + + Autoencode user if needed. + + Clear user/password if user is None. + + """ + # N.B. doesn't cleanup query/fragment + if user is None: + password = None + elif isinstance(user, str): + user = QUOTER(user) + password = self.raw_password + else: + raise TypeError("Invalid user type") + if not (netloc := self._netloc): + raise ValueError("user replacement is not allowed for relative URLs") + encoded_host = self.host_subcomponent or "" + netloc = make_netloc(user, password, encoded_host, self.explicit_port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_password(self, password: Union[str, None]) -> "URL": + """Return a new URL with password replaced. + + Autoencode password if needed. + + Clear password if argument is None. + + """ + # N.B. doesn't cleanup query/fragment + if password is None: + pass + elif isinstance(password, str): + password = QUOTER(password) + else: + raise TypeError("Invalid password type") + if not (netloc := self._netloc): + raise ValueError("password replacement is not allowed for relative URLs") + encoded_host = self.host_subcomponent or "" + port = self.explicit_port + netloc = make_netloc(self.raw_user, password, encoded_host, port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_host(self, host: str) -> "URL": + """Return a new URL with host replaced. + + Autoencode host if needed. + + Changing host for relative URLs is not allowed, use .join() + instead. + + """ + # N.B. doesn't cleanup query/fragment + if not isinstance(host, str): + raise TypeError("Invalid host type") + if not (netloc := self._netloc): + raise ValueError("host replacement is not allowed for relative URLs") + if not host: + raise ValueError("host removing is not allowed") + encoded_host = _encode_host(host, validate_host=True) if host else "" + port = self.explicit_port + netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_port(self, port: Union[int, None]) -> "URL": + """Return a new URL with port replaced. + + Clear port to default if None is passed. + + """ + # N.B. doesn't cleanup query/fragment + if port is not None: + if isinstance(port, bool) or not isinstance(port, int): + raise TypeError(f"port should be int or None, got {type(port)}") + if not (0 <= port <= 65535): + raise ValueError(f"port must be between 0 and 65535, got {port}") + if not (netloc := self._netloc): + raise ValueError("port replacement is not allowed for relative URLs") + encoded_host = self.host_subcomponent or "" + netloc = make_netloc(self.raw_user, self.raw_password, encoded_host, port) + return from_parts(self._scheme, netloc, self._path, self._query, self._fragment) + + def with_path( + self, + path: str, + *, + encoded: bool = False, + keep_query: bool = False, + keep_fragment: bool = False, + ) -> "URL": + """Return a new URL with path replaced.""" + netloc = self._netloc + if not encoded: + path = PATH_QUOTER(path) + if netloc: + path = normalize_path(path) if "." in path else path + if path and path[0] != "/": + path = f"/{path}" + query = self._query if keep_query else "" + fragment = self._fragment if keep_fragment else "" + return from_parts(self._scheme, netloc, path, query, fragment) + + @overload + def with_query(self, query: Query) -> "URL": ... + + @overload + def with_query(self, **kwargs: QueryVariable) -> "URL": ... + + def with_query(self, *args: Any, **kwargs: Any) -> "URL": + """Return a new URL with query part replaced. + + Accepts any Mapping (e.g. dict, multidict.MultiDict instances) + or str, autoencode the argument if needed. + + A sequence of (key, value) pairs is supported as well. + + It also can take an arbitrary number of keyword arguments. + + Clear query if None is passed. + + """ + # N.B. doesn't cleanup query/fragment + query = get_str_query(*args, **kwargs) or "" + return from_parts_uncached( + self._scheme, self._netloc, self._path, query, self._fragment + ) + + @overload + def extend_query(self, query: Query) -> "URL": ... + + @overload + def extend_query(self, **kwargs: QueryVariable) -> "URL": ... + + def extend_query(self, *args: Any, **kwargs: Any) -> "URL": + """Return a new URL with query part combined with the existing. + + This method will not remove existing query parameters. + + Example: + >>> url = URL('http://example.com/?a=1&b=2') + >>> url.extend_query(a=3, c=4) + URL('http://example.com/?a=1&b=2&a=3&c=4') + """ + if not (new_query := get_str_query(*args, **kwargs)): + return self + if query := self._query: + # both strings are already encoded so we can use a simple + # string join + query += new_query if query[-1] == "&" else f"&{new_query}" + else: + query = new_query + return from_parts_uncached( + self._scheme, self._netloc, self._path, query, self._fragment + ) + + @overload + def update_query(self, query: Query) -> "URL": ... + + @overload + def update_query(self, **kwargs: QueryVariable) -> "URL": ... + + def update_query(self, *args: Any, **kwargs: Any) -> "URL": + """Return a new URL with query part updated. + + This method will overwrite existing query parameters. + + Example: + >>> url = URL('http://example.com/?a=1&b=2') + >>> url.update_query(a=3, c=4) + URL('http://example.com/?a=3&b=2&c=4') + """ + in_query: Union[ + str, + Mapping[str, QueryVariable], + Sequence[tuple[Union[str, istr], SimpleQuery]], + None, + ] + if kwargs: + if args: + msg = "Either kwargs or single query parameter must be present" + raise ValueError(msg) + in_query = kwargs + elif len(args) == 1: + in_query = args[0] + else: + raise ValueError("Either kwargs or single query parameter must be present") + + if in_query is None: + query = "" + elif not in_query: + query = self._query + elif isinstance(in_query, Mapping): + qm: MultiDict[QueryVariable] = MultiDict(self._parsed_query) + qm.update(in_query) + query = get_str_query_from_sequence_iterable(qm.items()) + elif isinstance(in_query, str): + qstr: MultiDict[str] = MultiDict(self._parsed_query) + qstr.update(query_to_pairs(in_query)) + query = get_str_query_from_iterable(qstr.items()) + elif isinstance(in_query, (bytes, bytearray, memoryview)): + msg = "Invalid query type: bytes, bytearray and memoryview are forbidden" + raise TypeError(msg) + elif isinstance(in_query, Sequence): + # We don't expect sequence values if we're given a list of pairs + # already; only mappings like builtin `dict` which can't have the + # same key pointing to multiple values are allowed to use + # `_query_seq_pairs`. + if TYPE_CHECKING: + in_query = cast( + Sequence[tuple[Union[str, istr], SimpleQuery]], in_query + ) + qs: MultiDict[SimpleQuery] = MultiDict(self._parsed_query) + qs.update(in_query) + query = get_str_query_from_iterable(qs.items()) + else: + raise TypeError( + "Invalid query type: only str, mapping or " + "sequence of (key, value) pairs is allowed" + ) + return from_parts_uncached( + self._scheme, self._netloc, self._path, query, self._fragment + ) + + def without_query_params(self, *query_params: str) -> "URL": + """Remove some keys from query part and return new URL.""" + params_to_remove = set(query_params) & self.query.keys() + if not params_to_remove: + return self + return self.with_query( + tuple( + (name, value) + for name, value in self.query.items() + if name not in params_to_remove + ) + ) + + def with_fragment(self, fragment: Union[str, None]) -> "URL": + """Return a new URL with fragment replaced. + + Autoencode fragment if needed. + + Clear fragment to default if None is passed. + + """ + # N.B. doesn't cleanup query/fragment + if fragment is None: + raw_fragment = "" + elif not isinstance(fragment, str): + raise TypeError("Invalid fragment type") + else: + raw_fragment = FRAGMENT_QUOTER(fragment) + if self._fragment == raw_fragment: + return self + return from_parts( + self._scheme, self._netloc, self._path, self._query, raw_fragment + ) + + def with_name( + self, + name: str, + *, + keep_query: bool = False, + keep_fragment: bool = False, + ) -> "URL": + """Return a new URL with name (last part of path) replaced. + + Query and fragment parts are cleaned up. + + Name is encoded if needed. + + """ + # N.B. DOES cleanup query/fragment + if not isinstance(name, str): + raise TypeError("Invalid name type") + if "/" in name: + raise ValueError("Slash in name is not allowed") + name = PATH_QUOTER(name) + if name in (".", ".."): + raise ValueError(". and .. values are forbidden") + parts = list(self.raw_parts) + if netloc := self._netloc: + if len(parts) == 1: + parts.append(name) + else: + parts[-1] = name + parts[0] = "" # replace leading '/' + else: + parts[-1] = name + if parts[0] == "/": + parts[0] = "" # replace leading '/' + + query = self._query if keep_query else "" + fragment = self._fragment if keep_fragment else "" + return from_parts(self._scheme, netloc, "/".join(parts), query, fragment) + + def with_suffix( + self, + suffix: str, + *, + keep_query: bool = False, + keep_fragment: bool = False, + ) -> "URL": + """Return a new URL with suffix (file extension of name) replaced. + + Query and fragment parts are cleaned up. + + suffix is encoded if needed. + """ + if not isinstance(suffix, str): + raise TypeError("Invalid suffix type") + if suffix and not suffix[0] == "." or suffix == "." or "/" in suffix: + raise ValueError(f"Invalid suffix {suffix!r}") + name = self.raw_name + if not name: + raise ValueError(f"{self!r} has an empty name") + old_suffix = self.raw_suffix + suffix = PATH_QUOTER(suffix) + name = name + suffix if not old_suffix else name[: -len(old_suffix)] + suffix + if name in (".", ".."): + raise ValueError(". and .. values are forbidden") + parts = list(self.raw_parts) + if netloc := self._netloc: + if len(parts) == 1: + parts.append(name) + else: + parts[-1] = name + parts[0] = "" # replace leading '/' + else: + parts[-1] = name + if parts[0] == "/": + parts[0] = "" # replace leading '/' + + query = self._query if keep_query else "" + fragment = self._fragment if keep_fragment else "" + return from_parts(self._scheme, netloc, "/".join(parts), query, fragment) + + def join(self, url: "URL") -> "URL": + """Join URLs + + Construct a full (“absolute”) URL by combining a “base URL” + (self) with another URL (url). + + Informally, this uses components of the base URL, in + particular the addressing scheme, the network location and + (part of) the path, to provide missing components in the + relative URL. + + """ + if type(url) is not URL: + raise TypeError("url should be URL") + + scheme = url._scheme or self._scheme + if scheme != self._scheme or scheme not in USES_RELATIVE: + return url + + # scheme is in uses_authority as uses_authority is a superset of uses_relative + if (join_netloc := url._netloc) and scheme in USES_AUTHORITY: + return from_parts(scheme, join_netloc, url._path, url._query, url._fragment) + + orig_path = self._path + if join_path := url._path: + if join_path[0] == "/": + path = join_path + elif not orig_path: + path = f"/{join_path}" + elif orig_path[-1] == "/": + path = f"{orig_path}{join_path}" + else: + # … + # and relativizing ".." + # parts[0] is / for absolute urls, + # this join will add a double slash there + path = "/".join([*self.parts[:-1], ""]) + join_path + # which has to be removed + if orig_path[0] == "/": + path = path[1:] + path = normalize_path(path) if "." in path else path + else: + path = orig_path + + return from_parts( + scheme, + self._netloc, + path, + url._query if join_path or url._query else self._query, + url._fragment if join_path or url._fragment else self._fragment, + ) + + def joinpath(self, *other: str, encoded: bool = False) -> "URL": + """Return a new URL with the elements in other appended to the path.""" + return self._make_child(other, encoded=encoded) + + def human_repr(self) -> str: + """Return decoded human readable string for URL representation.""" + user = human_quote(self.user, "#/:?@[]") + password = human_quote(self.password, "#/:?@[]") + if (host := self.host) and ":" in host: + host = f"[{host}]" + path = human_quote(self.path, "#?") + if TYPE_CHECKING: + assert path is not None + query_string = "&".join( + "{}={}".format(human_quote(k, "#&+;="), human_quote(v, "#&+;=")) + for k, v in self.query.items() + ) + fragment = human_quote(self.fragment, "") + if TYPE_CHECKING: + assert fragment is not None + netloc = make_netloc(user, password, host, self.explicit_port) + return unsplit_result(self._scheme, netloc, path, query_string, fragment) + + +_DEFAULT_IDNA_SIZE = 256 +_DEFAULT_ENCODE_SIZE = 512 + + +@lru_cache(_DEFAULT_IDNA_SIZE) +def _idna_decode(raw: str) -> str: + try: + return idna.decode(raw.encode("ascii")) + except UnicodeError: # e.g. '::1' + return raw.encode("ascii").decode("idna") + + +@lru_cache(_DEFAULT_IDNA_SIZE) +def _idna_encode(host: str) -> str: + try: + return idna.encode(host, uts46=True).decode("ascii") + except UnicodeError: + return host.encode("idna").decode("ascii") + + +@lru_cache(_DEFAULT_ENCODE_SIZE) +def _encode_host(host: str, validate_host: bool) -> str: + """Encode host part of URL.""" + # If the host ends with a digit or contains a colon, its likely + # an IP address. + if host and (host[-1].isdigit() or ":" in host): + raw_ip, sep, zone = host.partition("%") + # If it looks like an IP, we check with _ip_compressed_version + # and fall-through if its not an IP address. This is a performance + # optimization to avoid parsing IP addresses as much as possible + # because it is orders of magnitude slower than almost any other + # operation this library does. + # Might be an IP address, check it + # + # IP Addresses can look like: + # https://datatracker.ietf.org/doc/html/rfc3986#section-3.2.2 + # - 127.0.0.1 (last character is a digit) + # - 2001:db8::ff00:42:8329 (contains a colon) + # - 2001:db8::ff00:42:8329%eth0 (contains a colon) + # - [2001:db8::ff00:42:8329] (contains a colon -- brackets should + # have been removed before it gets here) + # Rare IP Address formats are not supported per: + # https://datatracker.ietf.org/doc/html/rfc3986#section-7.4 + # + # IP parsing is slow, so its wrapped in an LRU + try: + ip = ip_address(raw_ip) + except ValueError: + pass + else: + # These checks should not happen in the + # LRU to keep the cache size small + host = ip.compressed + if ip.version == 6: + return f"[{host}%{zone}]" if sep else f"[{host}]" + return f"{host}%{zone}" if sep else host + + # IDNA encoding is slow, skip it for ASCII-only strings + if host.isascii(): + # Check for invalid characters explicitly; _idna_encode() does this + # for non-ascii host names. + host = host.lower() + if validate_host and (invalid := NOT_REG_NAME.search(host)): + value, pos, extra = invalid.group(), invalid.start(), "" + if value == "@" or (value == ":" and "@" in host[pos:]): + # this looks like an authority string + extra = ( + ", if the value includes a username or password, " + "use 'authority' instead of 'host'" + ) + raise ValueError( + f"Host {host!r} cannot contain {value!r} (at position {pos}){extra}" + ) from None + return host + + return _idna_encode(host) + + +@rewrite_module +def cache_clear() -> None: + """Clear all LRU caches.""" + _idna_encode.cache_clear() + _idna_decode.cache_clear() + _encode_host.cache_clear() + + +@rewrite_module +def cache_info() -> CacheInfo: + """Report cache statistics.""" + return { + "idna_encode": _idna_encode.cache_info(), + "idna_decode": _idna_decode.cache_info(), + "ip_address": _encode_host.cache_info(), + "host_validate": _encode_host.cache_info(), + "encode_host": _encode_host.cache_info(), + } + + +@rewrite_module +def cache_configure( + *, + idna_encode_size: Union[int, None] = _DEFAULT_IDNA_SIZE, + idna_decode_size: Union[int, None] = _DEFAULT_IDNA_SIZE, + ip_address_size: Union[int, None, UndefinedType] = UNDEFINED, + host_validate_size: Union[int, None, UndefinedType] = UNDEFINED, + encode_host_size: Union[int, None, UndefinedType] = UNDEFINED, +) -> None: + """Configure LRU cache sizes.""" + global _idna_decode, _idna_encode, _encode_host + # ip_address_size, host_validate_size are no longer + # used, but are kept for backwards compatibility. + if ip_address_size is not UNDEFINED or host_validate_size is not UNDEFINED: + warnings.warn( + "cache_configure() no longer accepts the " + "ip_address_size or host_validate_size arguments, " + "they are used to set the encode_host_size instead " + "and will be removed in the future", + DeprecationWarning, + stacklevel=2, + ) + + if encode_host_size is not None: + for size in (ip_address_size, host_validate_size): + if size is None: + encode_host_size = None + elif encode_host_size is UNDEFINED: + if size is not UNDEFINED: + encode_host_size = size + elif size is not UNDEFINED: + if TYPE_CHECKING: + assert isinstance(size, int) + assert isinstance(encode_host_size, int) + encode_host_size = max(size, encode_host_size) + if encode_host_size is UNDEFINED: + encode_host_size = _DEFAULT_ENCODE_SIZE + + _encode_host = lru_cache(encode_host_size)(_encode_host.__wrapped__) + _idna_decode = lru_cache(idna_decode_size)(_idna_decode.__wrapped__) + _idna_encode = lru_cache(idna_encode_size)(_idna_encode.__wrapped__) diff --git a/lib/python3.12/site-packages/yarl/py.typed b/lib/python3.12/site-packages/yarl/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..dcf2c804da5e19d617a03a6c68aa128d1d1f89a0 --- /dev/null +++ b/lib/python3.12/site-packages/yarl/py.typed @@ -0,0 +1 @@ +# Placeholder