diff --git a/lib/python3.12/site-packages/fsspec/__init__.py b/lib/python3.12/site-packages/fsspec/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..d7af775f77cd67734aebd25ce4e051be87e64fa8 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/__init__.py @@ -0,0 +1,69 @@ +from importlib.metadata import entry_points + +from . import caching +from ._version import __version__ # noqa: F401 +from .callbacks import Callback +from .compression import available_compressions +from .core import get_fs_token_paths, open, open_files, open_local, url_to_fs +from .exceptions import FSTimeoutError +from .mapping import FSMap, get_mapper +from .registry import ( + available_protocols, + filesystem, + get_filesystem_class, + register_implementation, + registry, +) +from .spec import AbstractFileSystem + +__all__ = [ + "AbstractFileSystem", + "FSTimeoutError", + "FSMap", + "filesystem", + "register_implementation", + "get_filesystem_class", + "get_fs_token_paths", + "get_mapper", + "open", + "open_files", + "open_local", + "registry", + "caching", + "Callback", + "available_protocols", + "available_compressions", + "url_to_fs", +] + + +def process_entries(): + if entry_points is not None: + try: + eps = entry_points() + except TypeError: + pass # importlib-metadata < 0.8 + else: + if hasattr(eps, "select"): # Python 3.10+ / importlib_metadata >= 3.9.0 + specs = eps.select(group="fsspec.specs") + else: + specs = eps.get("fsspec.specs", []) + registered_names = {} + for spec in specs: + err_msg = f"Unable to load filesystem from {spec}" + name = spec.name + if name in registered_names: + continue + registered_names[name] = True + register_implementation( + name, + spec.value.replace(":", "."), + errtxt=err_msg, + # We take our implementations as the ones to overload with if + # for some reason we encounter some, may be the same, already + # registered + clobber=True, + ) + + +process_entries() diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d870ed538f300edb7617c0af922eef5d0e81bca Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/_version.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/_version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..35abfb0b77c76cf81df3d99d3aff6c29a63a09a7 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/_version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/archive.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/archive.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6eb914def5d9916fc204cf3687735088a4871707 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/archive.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/asyn.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/asyn.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a8935c01cdf92a382a43f909d57f5758f19ad6de Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/asyn.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/caching.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/caching.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..032c087df035427599413a5e55eb5a293db63c3b Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/caching.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/callbacks.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/callbacks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec0d2bd5990c7ce3c32cf7a709dcc972f5547f25 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/callbacks.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/compression.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/compression.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..33cd7d1801a020877f732fbd56ead039bcb197ee Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/compression.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/config.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/config.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cf0b461cef9771318745a921c4aa0054557cfa91 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/config.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/conftest.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/conftest.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d4d788515a98d01bdc9abaa35364624da83bd6ec Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/conftest.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/core.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f005630cd6e62685017ca436019469d589d11dc1 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/core.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/dircache.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/dircache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a4d1318e3b36b30ee89574c228e28311d6e969a Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/dircache.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/exceptions.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..219c6c60f0a443dc9354a4b23440fce293492fef Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/exceptions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/fuse.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/fuse.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2f215a491996c16575e21f665aef75e920bde565 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/fuse.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/generic.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/generic.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5654e4b4ec5c3a838f5ef3405f7bf67bed2ac810 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/generic.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/gui.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/gui.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1debefba2931f8d41b2606675a06f1cf7e10af99 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/gui.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/json.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/json.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4186c0de44df19e9f4833d879efda85489c24d61 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/json.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/mapping.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/mapping.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..894d95346e388658c087f86751b6b4642ad80667 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/mapping.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/parquet.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/parquet.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb3fcb78cb97f3b2a26a260eea964c91edbf3e22 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/parquet.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/registry.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/registry.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8c4b8a4cfbede978fa060884aef0825eff0fe357 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/registry.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/spec.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/spec.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49491eb49a95bd7a382f6d38e7031230142e93c2 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/spec.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/transaction.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/transaction.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f63a8e5556f4054c91aeec6df029c511abe670ab Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/transaction.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/__pycache__/utils.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5c6570d86aaf71d1f21d764cb6064bcd596b7b94 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/__pycache__/utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/_version.py b/lib/python3.12/site-packages/fsspec/_version.py new file mode 100644 index 0000000000000000000000000000000000000000..0de14558d32cb1aaa09200485d1049f968a00e0b --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/_version.py @@ -0,0 +1,21 @@ +# file generated by setuptools-scm +# don't change, don't track in version control + +__all__ = ["__version__", "__version_tuple__", "version", "version_tuple"] + +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple + from typing import Union + + VERSION_TUPLE = Tuple[Union[int, str], ...] +else: + VERSION_TUPLE = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE + +__version__ = version = '2025.3.0' +__version_tuple__ = version_tuple = (2025, 3, 0) diff --git a/lib/python3.12/site-packages/fsspec/archive.py b/lib/python3.12/site-packages/fsspec/archive.py new file mode 100644 index 0000000000000000000000000000000000000000..13a4da8df7c9405297cdd7d37476be2f725b2f57 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/archive.py @@ -0,0 +1,75 @@ +import operator + +from fsspec import AbstractFileSystem +from fsspec.utils import tokenize + + +class AbstractArchiveFileSystem(AbstractFileSystem): + """ + A generic superclass for implementing Archive-based filesystems. + + Currently, it is shared amongst + :class:`~fsspec.implementations.zip.ZipFileSystem`, + :class:`~fsspec.implementations.libarchive.LibArchiveFileSystem` and + :class:`~fsspec.implementations.tar.TarFileSystem`. + """ + + def __str__(self): + return f"" + + __repr__ = __str__ + + def ukey(self, path): + return tokenize(path, self.fo, self.protocol) + + def _all_dirnames(self, paths): + """Returns *all* directory names for each path in paths, including intermediate + ones. + + Parameters + ---------- + paths: Iterable of path strings + """ + if len(paths) == 0: + return set() + + dirnames = {self._parent(path) for path in paths} - {self.root_marker} + return dirnames | self._all_dirnames(dirnames) + + def info(self, path, **kwargs): + self._get_dirs() + path = self._strip_protocol(path) + if path in {"", "/"} and self.dir_cache: + return {"name": "", "type": "directory", "size": 0} + if path in self.dir_cache: + return self.dir_cache[path] + elif path + "/" in self.dir_cache: + return self.dir_cache[path + "/"] + else: + raise FileNotFoundError(path) + + def ls(self, path, detail=True, **kwargs): + self._get_dirs() + paths = {} + for p, f in self.dir_cache.items(): + p = p.rstrip("/") + if "/" in p: + root = p.rsplit("/", 1)[0] + else: + root = "" + if root == path.rstrip("/"): + paths[p] = f + elif all( + (a == b) + for a, b in zip(path.split("/"), [""] + p.strip("/").split("/")) + ): + # root directory entry + ppath = p.rstrip("/").split("/", 1)[0] + if ppath not in paths: + out = {"name": ppath, "size": 0, "type": "directory"} + paths[ppath] = out + if detail: + out = sorted(paths.values(), key=operator.itemgetter("name")) + return out + else: + return sorted(paths) diff --git a/lib/python3.12/site-packages/fsspec/asyn.py b/lib/python3.12/site-packages/fsspec/asyn.py new file mode 100644 index 0000000000000000000000000000000000000000..adeb88fb4c72932457ac31e0ff0ff5c26438e317 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/asyn.py @@ -0,0 +1,1098 @@ +import asyncio +import asyncio.events +import functools +import inspect +import io +import numbers +import os +import re +import threading +from contextlib import contextmanager +from glob import has_magic +from typing import TYPE_CHECKING, Iterable + +from .callbacks import DEFAULT_CALLBACK +from .exceptions import FSTimeoutError +from .implementations.local import LocalFileSystem, make_path_posix, trailing_sep +from .spec import AbstractBufferedFile, AbstractFileSystem +from .utils import glob_translate, is_exception, other_paths + +private = re.compile("_[^_]") +iothread = [None] # dedicated fsspec IO thread +loop = [None] # global event loop for any non-async instance +_lock = None # global lock placeholder +get_running_loop = asyncio.get_running_loop + + +def get_lock(): + """Allocate or return a threading lock. + + The lock is allocated on first use to allow setting one lock per forked process. + """ + global _lock + if not _lock: + _lock = threading.Lock() + return _lock + + +def reset_lock(): + """Reset the global lock. + + This should be called only on the init of a forked process to reset the lock to + None, enabling the new forked process to get a new lock. + """ + global _lock + + iothread[0] = None + loop[0] = None + _lock = None + + +async def _runner(event, coro, result, timeout=None): + timeout = timeout if timeout else None # convert 0 or 0.0 to None + if timeout is not None: + coro = asyncio.wait_for(coro, timeout=timeout) + try: + result[0] = await coro + except Exception as ex: + result[0] = ex + finally: + event.set() + + +def sync(loop, func, *args, timeout=None, **kwargs): + """ + Make loop run coroutine until it returns. Runs in other thread + + Examples + -------- + >>> fsspec.asyn.sync(fsspec.asyn.get_loop(), func, *args, + timeout=timeout, **kwargs) + """ + timeout = timeout if timeout else None # convert 0 or 0.0 to None + # NB: if the loop is not running *yet*, it is OK to submit work + # and we will wait for it + if loop is None or loop.is_closed(): + raise RuntimeError("Loop is not running") + try: + loop0 = asyncio.events.get_running_loop() + if loop0 is loop: + raise NotImplementedError("Calling sync() from within a running loop") + except NotImplementedError: + raise + except RuntimeError: + pass + coro = func(*args, **kwargs) + result = [None] + event = threading.Event() + asyncio.run_coroutine_threadsafe(_runner(event, coro, result, timeout), loop) + while True: + # this loops allows thread to get interrupted + if event.wait(1): + break + if timeout is not None: + timeout -= 1 + if timeout < 0: + raise FSTimeoutError + + return_result = result[0] + if isinstance(return_result, asyncio.TimeoutError): + # suppress asyncio.TimeoutError, raise FSTimeoutError + raise FSTimeoutError from return_result + elif isinstance(return_result, BaseException): + raise return_result + else: + return return_result + + +def sync_wrapper(func, obj=None): + """Given a function, make so can be called in blocking contexts + + Leave obj=None if defining within a class. Pass the instance if attaching + as an attribute of the instance. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + self = obj or args[0] + return sync(self.loop, func, *args, **kwargs) + + return wrapper + + +@contextmanager +def _selector_policy(): + original_policy = asyncio.get_event_loop_policy() + try: + if os.name == "nt" and hasattr(asyncio, "WindowsSelectorEventLoopPolicy"): + asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) + + yield + finally: + asyncio.set_event_loop_policy(original_policy) + + +def get_loop(): + """Create or return the default fsspec IO loop + + The loop will be running on a separate thread. + """ + if loop[0] is None: + with get_lock(): + # repeat the check just in case the loop got filled between the + # previous two calls from another thread + if loop[0] is None: + with _selector_policy(): + loop[0] = asyncio.new_event_loop() + th = threading.Thread(target=loop[0].run_forever, name="fsspecIO") + th.daemon = True + th.start() + iothread[0] = th + return loop[0] + + +if TYPE_CHECKING: + import resource + + ResourceError = resource.error +else: + try: + import resource + except ImportError: + resource = None + ResourceError = OSError + else: + ResourceError = getattr(resource, "error", OSError) + +_DEFAULT_BATCH_SIZE = 128 +_NOFILES_DEFAULT_BATCH_SIZE = 1280 + + +def _get_batch_size(nofiles=False): + from fsspec.config import conf + + if nofiles: + if "nofiles_gather_batch_size" in conf: + return conf["nofiles_gather_batch_size"] + else: + if "gather_batch_size" in conf: + return conf["gather_batch_size"] + if nofiles: + return _NOFILES_DEFAULT_BATCH_SIZE + if resource is None: + return _DEFAULT_BATCH_SIZE + + try: + soft_limit, _ = resource.getrlimit(resource.RLIMIT_NOFILE) + except (ImportError, ValueError, ResourceError): + return _DEFAULT_BATCH_SIZE + + if soft_limit == resource.RLIM_INFINITY: + return -1 + else: + return soft_limit // 8 + + +def running_async() -> bool: + """Being executed by an event loop?""" + try: + asyncio.get_running_loop() + return True + except RuntimeError: + return False + + +async def _run_coros_in_chunks( + coros, + batch_size=None, + callback=DEFAULT_CALLBACK, + timeout=None, + return_exceptions=False, + nofiles=False, +): + """Run the given coroutines in chunks. + + Parameters + ---------- + coros: list of coroutines to run + batch_size: int or None + Number of coroutines to submit/wait on simultaneously. + If -1, then it will not be any throttling. If + None, it will be inferred from _get_batch_size() + callback: fsspec.callbacks.Callback instance + Gets a relative_update when each coroutine completes + timeout: number or None + If given, each coroutine times out after this time. Note that, since + there are multiple batches, the total run time of this function will in + general be longer + return_exceptions: bool + Same meaning as in asyncio.gather + nofiles: bool + If inferring the batch_size, does this operation involve local files? + If yes, you normally expect smaller batches. + """ + + if batch_size is None: + batch_size = _get_batch_size(nofiles=nofiles) + + if batch_size == -1: + batch_size = len(coros) + + assert batch_size > 0 + + async def _run_coro(coro, i): + try: + return await asyncio.wait_for(coro, timeout=timeout), i + except Exception as e: + if not return_exceptions: + raise + return e, i + finally: + callback.relative_update(1) + + i = 0 + n = len(coros) + results = [None] * n + pending = set() + + while pending or i < n: + while len(pending) < batch_size and i < n: + pending.add(asyncio.ensure_future(_run_coro(coros[i], i))) + i += 1 + + if not pending: + break + + done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED) + while done: + result, k = await done.pop() + results[k] = result + + return results + + +# these methods should be implemented as async by any async-able backend +async_methods = [ + "_ls", + "_cat_file", + "_get_file", + "_put_file", + "_rm_file", + "_cp_file", + "_pipe_file", + "_expand_path", + "_info", + "_isfile", + "_isdir", + "_exists", + "_walk", + "_glob", + "_find", + "_du", + "_size", + "_mkdir", + "_makedirs", +] + + +class AsyncFileSystem(AbstractFileSystem): + """Async file operations, default implementations + + Passes bulk operations to asyncio.gather for concurrent operation. + + Implementations that have concurrent batch operations and/or async methods + should inherit from this class instead of AbstractFileSystem. Docstrings are + copied from the un-underscored method in AbstractFileSystem, if not given. + """ + + # note that methods do not have docstring here; they will be copied + # for _* methods and inferred for overridden methods. + + async_impl = True + mirror_sync_methods = True + disable_throttling = False + + def __init__(self, *args, asynchronous=False, loop=None, batch_size=None, **kwargs): + self.asynchronous = asynchronous + self._pid = os.getpid() + if not asynchronous: + self._loop = loop or get_loop() + else: + self._loop = None + self.batch_size = batch_size + super().__init__(*args, **kwargs) + + @property + def loop(self): + if self._pid != os.getpid(): + raise RuntimeError("This class is not fork-safe") + return self._loop + + async def _rm_file(self, path, **kwargs): + raise NotImplementedError + + async def _rm(self, path, recursive=False, batch_size=None, **kwargs): + # TODO: implement on_error + batch_size = batch_size or self.batch_size + path = await self._expand_path(path, recursive=recursive) + return await _run_coros_in_chunks( + [self._rm_file(p, **kwargs) for p in reversed(path)], + batch_size=batch_size, + nofiles=True, + ) + + async def _cp_file(self, path1, path2, **kwargs): + raise NotImplementedError + + async def _mv_file(self, path1, path2): + await self._cp_file(path1, path2) + await self._rm_file(path1) + + async def _copy( + self, + path1, + path2, + recursive=False, + on_error=None, + maxdepth=None, + batch_size=None, + **kwargs, + ): + if on_error is None and recursive: + on_error = "ignore" + elif on_error is None: + on_error = "raise" + + if isinstance(path1, list) and isinstance(path2, list): + # No need to expand paths when both source and destination + # are provided as lists + paths1 = path1 + paths2 = path2 + else: + source_is_str = isinstance(path1, str) + paths1 = await self._expand_path( + path1, maxdepth=maxdepth, recursive=recursive + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + paths1 = [ + p for p in paths1 if not (trailing_sep(p) or await self._isdir(p)) + ] + if not paths1: + return + + source_is_file = len(paths1) == 1 + dest_is_dir = isinstance(path2, str) and ( + trailing_sep(path2) or await self._isdir(path2) + ) + + exists = source_is_str and ( + (has_magic(path1) and source_is_file) + or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1)) + ) + paths2 = other_paths( + paths1, + path2, + exists=exists, + flatten=not source_is_str, + ) + + batch_size = batch_size or self.batch_size + coros = [self._cp_file(p1, p2, **kwargs) for p1, p2 in zip(paths1, paths2)] + result = await _run_coros_in_chunks( + coros, batch_size=batch_size, return_exceptions=True, nofiles=True + ) + + for ex in filter(is_exception, result): + if on_error == "ignore" and isinstance(ex, FileNotFoundError): + continue + raise ex + + async def _pipe_file(self, path, value, mode="overwrite", **kwargs): + raise NotImplementedError + + async def _pipe(self, path, value=None, batch_size=None, **kwargs): + if isinstance(path, str): + path = {path: value} + batch_size = batch_size or self.batch_size + return await _run_coros_in_chunks( + [self._pipe_file(k, v, **kwargs) for k, v in path.items()], + batch_size=batch_size, + nofiles=True, + ) + + async def _process_limits(self, url, start, end): + """Helper for "Range"-based _cat_file""" + size = None + suff = False + if start is not None and start < 0: + # if start is negative and end None, end is the "suffix length" + if end is None: + end = -start + start = "" + suff = True + else: + size = size or (await self._info(url))["size"] + start = size + start + elif start is None: + start = 0 + if not suff: + if end is not None and end < 0: + if start is not None: + size = size or (await self._info(url))["size"] + end = size + end + elif end is None: + end = "" + if isinstance(end, numbers.Integral): + end -= 1 # bytes range is inclusive + return f"bytes={start}-{end}" + + async def _cat_file(self, path, start=None, end=None, **kwargs): + raise NotImplementedError + + async def _cat( + self, path, recursive=False, on_error="raise", batch_size=None, **kwargs + ): + paths = await self._expand_path(path, recursive=recursive) + coros = [self._cat_file(path, **kwargs) for path in paths] + batch_size = batch_size or self.batch_size + out = await _run_coros_in_chunks( + coros, batch_size=batch_size, nofiles=True, return_exceptions=True + ) + if on_error == "raise": + ex = next(filter(is_exception, out), False) + if ex: + raise ex + if ( + len(paths) > 1 + or isinstance(path, list) + or paths[0] != self._strip_protocol(path) + ): + return { + k: v + for k, v in zip(paths, out) + if on_error != "omit" or not is_exception(v) + } + else: + return out[0] + + async def _cat_ranges( + self, + paths, + starts, + ends, + max_gap=None, + batch_size=None, + on_error="return", + **kwargs, + ): + """Get the contents of byte ranges from one or more files + + Parameters + ---------- + paths: list + A list of of filepaths on this filesystems + starts, ends: int or list + Bytes limits of the read. If using a single int, the same value will be + used to read all the specified files. + """ + # TODO: on_error + if max_gap is not None: + # use utils.merge_offset_ranges + raise NotImplementedError + if not isinstance(paths, list): + raise TypeError + if not isinstance(starts, Iterable): + starts = [starts] * len(paths) + if not isinstance(ends, Iterable): + ends = [ends] * len(paths) + if len(starts) != len(paths) or len(ends) != len(paths): + raise ValueError + coros = [ + self._cat_file(p, start=s, end=e, **kwargs) + for p, s, e in zip(paths, starts, ends) + ] + batch_size = batch_size or self.batch_size + return await _run_coros_in_chunks( + coros, batch_size=batch_size, nofiles=True, return_exceptions=True + ) + + async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs): + raise NotImplementedError + + async def _put( + self, + lpath, + rpath, + recursive=False, + callback=DEFAULT_CALLBACK, + batch_size=None, + maxdepth=None, + **kwargs, + ): + """Copy file(s) from local. + + Copies a specific file or tree of files (if recursive=True). If rpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. + + The put_file method will be called concurrently on a batch of files. The + batch_size option can configure the amount of futures that can be executed + at the same time. If it is -1, then all the files will be uploaded concurrently. + The default can be set for this instance by passing "batch_size" in the + constructor, or for all instances by setting the "gather_batch_size" key + in ``fsspec.config.conf``, falling back to 1/8th of the system limit . + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + source_is_str = isinstance(lpath, str) + if source_is_str: + lpath = make_path_posix(lpath) + fs = LocalFileSystem() + lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))] + if not lpaths: + return + + source_is_file = len(lpaths) == 1 + dest_is_dir = isinstance(rpath, str) and ( + trailing_sep(rpath) or await self._isdir(rpath) + ) + + rpath = self._strip_protocol(rpath) + exists = source_is_str and ( + (has_magic(lpath) and source_is_file) + or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) + ) + rpaths = other_paths( + lpaths, + rpath, + exists=exists, + flatten=not source_is_str, + ) + + is_dir = {l: os.path.isdir(l) for l in lpaths} + rdirs = [r for l, r in zip(lpaths, rpaths) if is_dir[l]] + file_pairs = [(l, r) for l, r in zip(lpaths, rpaths) if not is_dir[l]] + + await asyncio.gather(*[self._makedirs(d, exist_ok=True) for d in rdirs]) + batch_size = batch_size or self.batch_size + + coros = [] + callback.set_size(len(file_pairs)) + for lfile, rfile in file_pairs: + put_file = callback.branch_coro(self._put_file) + coros.append(put_file(lfile, rfile, **kwargs)) + + return await _run_coros_in_chunks( + coros, batch_size=batch_size, callback=callback + ) + + async def _get_file(self, rpath, lpath, **kwargs): + raise NotImplementedError + + async def _get( + self, + rpath, + lpath, + recursive=False, + callback=DEFAULT_CALLBACK, + maxdepth=None, + **kwargs, + ): + """Copy file(s) to local. + + Copies a specific file or tree of files (if recursive=True). If lpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. Can submit a list of paths, which may be glob-patterns + and will be expanded. + + The get_file method will be called concurrently on a batch of files. The + batch_size option can configure the amount of futures that can be executed + at the same time. If it is -1, then all the files will be uploaded concurrently. + The default can be set for this instance by passing "batch_size" in the + constructor, or for all instances by setting the "gather_batch_size" key + in ``fsspec.config.conf``, falling back to 1/8th of the system limit . + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + source_is_str = isinstance(rpath, str) + # First check for rpath trailing slash as _strip_protocol removes it. + source_not_trailing_sep = source_is_str and not trailing_sep(rpath) + rpath = self._strip_protocol(rpath) + rpaths = await self._expand_path( + rpath, recursive=recursive, maxdepth=maxdepth + ) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + rpaths = [ + p for p in rpaths if not (trailing_sep(p) or await self._isdir(p)) + ] + if not rpaths: + return + + lpath = make_path_posix(lpath) + source_is_file = len(rpaths) == 1 + dest_is_dir = isinstance(lpath, str) and ( + trailing_sep(lpath) or LocalFileSystem().isdir(lpath) + ) + + exists = source_is_str and ( + (has_magic(rpath) and source_is_file) + or (not has_magic(rpath) and dest_is_dir and source_not_trailing_sep) + ) + lpaths = other_paths( + rpaths, + lpath, + exists=exists, + flatten=not source_is_str, + ) + + [os.makedirs(os.path.dirname(lp), exist_ok=True) for lp in lpaths] + batch_size = kwargs.pop("batch_size", self.batch_size) + + coros = [] + callback.set_size(len(lpaths)) + for lpath, rpath in zip(lpaths, rpaths): + get_file = callback.branch_coro(self._get_file) + coros.append(get_file(rpath, lpath, **kwargs)) + return await _run_coros_in_chunks( + coros, batch_size=batch_size, callback=callback + ) + + async def _isfile(self, path): + try: + return (await self._info(path))["type"] == "file" + except: # noqa: E722 + return False + + async def _isdir(self, path): + try: + return (await self._info(path))["type"] == "directory" + except OSError: + return False + + async def _size(self, path): + return (await self._info(path)).get("size", None) + + async def _sizes(self, paths, batch_size=None): + batch_size = batch_size or self.batch_size + return await _run_coros_in_chunks( + [self._size(p) for p in paths], batch_size=batch_size + ) + + async def _exists(self, path, **kwargs): + try: + await self._info(path, **kwargs) + return True + except FileNotFoundError: + return False + + async def _info(self, path, **kwargs): + raise NotImplementedError + + async def _ls(self, path, detail=True, **kwargs): + raise NotImplementedError + + async def _walk(self, path, maxdepth=None, on_error="omit", **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + path = self._strip_protocol(path) + full_dirs = {} + dirs = {} + files = {} + + detail = kwargs.pop("detail", False) + try: + listing = await self._ls(path, detail=True, **kwargs) + except (FileNotFoundError, OSError) as e: + if on_error == "raise": + raise + elif callable(on_error): + on_error(e) + if detail: + yield path, {}, {} + else: + yield path, [], [] + return + + for info in listing: + # each info name must be at least [path]/part , but here + # we check also for names like [path]/part/ + pathname = info["name"].rstrip("/") + name = pathname.rsplit("/", 1)[-1] + if info["type"] == "directory" and pathname != path: + # do not include "self" path + full_dirs[name] = pathname + dirs[name] = info + elif pathname == path: + # file-like with same name as give path + files[""] = info + else: + files[name] = info + + if detail: + yield path, dirs, files + else: + yield path, list(dirs), list(files) + + if maxdepth is not None: + maxdepth -= 1 + if maxdepth < 1: + return + + for d in dirs: + async for _ in self._walk( + full_dirs[d], maxdepth=maxdepth, detail=detail, **kwargs + ): + yield _ + + async def _glob(self, path, maxdepth=None, **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + import re + + seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) + ends_with_sep = path.endswith(seps) # _strip_protocol strips trailing slash + path = self._strip_protocol(path) + append_slash_to_dirname = ends_with_sep or path.endswith( + tuple(sep + "**" for sep in seps) + ) + idx_star = path.find("*") if path.find("*") >= 0 else len(path) + idx_qmark = path.find("?") if path.find("?") >= 0 else len(path) + idx_brace = path.find("[") if path.find("[") >= 0 else len(path) + + min_idx = min(idx_star, idx_qmark, idx_brace) + + detail = kwargs.pop("detail", False) + + if not has_magic(path): + if await self._exists(path, **kwargs): + if not detail: + return [path] + else: + return {path: await self._info(path, **kwargs)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:min_idx]: + min_idx = path[:min_idx].rindex("/") + root = path[: min_idx + 1] + depth = path[min_idx + 1 :].count("/") + 1 + else: + root = "" + depth = path[min_idx + 1 :].count("/") + 1 + + if "**" in path: + if maxdepth is not None: + idx_double_stars = path.find("**") + depth_double_stars = path[idx_double_stars:].count("/") + 1 + depth = depth - depth_double_stars + maxdepth + else: + depth = None + + allpaths = await self._find( + root, maxdepth=depth, withdirs=True, detail=True, **kwargs + ) + + pattern = glob_translate(path + ("/" if ends_with_sep else "")) + pattern = re.compile(pattern) + + out = { + p: info + for p, info in sorted(allpaths.items()) + if pattern.match( + p + "/" + if append_slash_to_dirname and info["type"] == "directory" + else p + ) + } + + if detail: + return out + else: + return list(out) + + async def _du(self, path, total=True, maxdepth=None, **kwargs): + sizes = {} + # async for? + for f in await self._find(path, maxdepth=maxdepth, **kwargs): + info = await self._info(f) + sizes[info["name"]] = info["size"] + if total: + return sum(sizes.values()) + else: + return sizes + + async def _find(self, path, maxdepth=None, withdirs=False, **kwargs): + path = self._strip_protocol(path) + out = {} + detail = kwargs.pop("detail", False) + + # Add the root directory if withdirs is requested + # This is needed for posix glob compliance + if withdirs and path != "" and await self._isdir(path): + out[path] = await self._info(path) + + # async for? + async for _, dirs, files in self._walk(path, maxdepth, detail=True, **kwargs): + if withdirs: + files.update(dirs) + out.update({info["name"]: info for name, info in files.items()}) + if not out and (await self._isfile(path)): + # walk works on directories, but find should also return [path] + # when path happens to be a file + out[path] = {} + names = sorted(out) + if not detail: + return names + else: + return {name: out[name] for name in names} + + async def _expand_path(self, path, recursive=False, maxdepth=None): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + if isinstance(path, str): + out = await self._expand_path([path], recursive, maxdepth) + else: + out = set() + path = [self._strip_protocol(p) for p in path] + for p in path: # can gather here + if has_magic(p): + bit = set(await self._glob(p, maxdepth=maxdepth)) + out |= bit + if recursive: + # glob call above expanded one depth so if maxdepth is defined + # then decrement it in expand_path call below. If it is zero + # after decrementing then avoid expand_path call. + if maxdepth is not None and maxdepth <= 1: + continue + out |= set( + await self._expand_path( + list(bit), + recursive=recursive, + maxdepth=maxdepth - 1 if maxdepth is not None else None, + ) + ) + continue + elif recursive: + rec = set(await self._find(p, maxdepth=maxdepth, withdirs=True)) + out |= rec + if p not in out and (recursive is False or (await self._exists(p))): + # should only check once, for the root + out.add(p) + if not out: + raise FileNotFoundError(path) + return sorted(out) + + async def _mkdir(self, path, create_parents=True, **kwargs): + pass # not necessary to implement, may not have directories + + async def _makedirs(self, path, exist_ok=False): + pass # not necessary to implement, may not have directories + + async def open_async(self, path, mode="rb", **kwargs): + if "b" not in mode or kwargs.get("compression"): + raise ValueError + raise NotImplementedError + + +def mirror_sync_methods(obj): + """Populate sync and async methods for obj + + For each method will create a sync version if the name refers to an async method + (coroutine) and there is no override in the child class; will create an async + method for the corresponding sync method if there is no implementation. + + Uses the methods specified in + - async_methods: the set that an implementation is expected to provide + - default_async_methods: that can be derived from their sync version in + AbstractFileSystem + - AsyncFileSystem: async-specific default coroutines + """ + from fsspec import AbstractFileSystem + + for method in async_methods + dir(AsyncFileSystem): + if not method.startswith("_"): + continue + smethod = method[1:] + if private.match(method): + isco = inspect.iscoroutinefunction(getattr(obj, method, None)) + unsync = getattr(getattr(obj, smethod, False), "__func__", None) + is_default = unsync is getattr(AbstractFileSystem, smethod, "") + if isco and is_default: + mth = sync_wrapper(getattr(obj, method), obj=obj) + setattr(obj, smethod, mth) + if not mth.__doc__: + mth.__doc__ = getattr( + getattr(AbstractFileSystem, smethod, None), "__doc__", "" + ) + + +class FSSpecCoroutineCancel(Exception): + pass + + +def _dump_running_tasks( + printout=True, cancel=True, exc=FSSpecCoroutineCancel, with_task=False +): + import traceback + + tasks = [t for t in asyncio.tasks.all_tasks(loop[0]) if not t.done()] + if printout: + [task.print_stack() for task in tasks] + out = [ + { + "locals": task._coro.cr_frame.f_locals, + "file": task._coro.cr_frame.f_code.co_filename, + "firstline": task._coro.cr_frame.f_code.co_firstlineno, + "linelo": task._coro.cr_frame.f_lineno, + "stack": traceback.format_stack(task._coro.cr_frame), + "task": task if with_task else None, + } + for task in tasks + ] + if cancel: + for t in tasks: + cbs = t._callbacks + t.cancel() + asyncio.futures.Future.set_exception(t, exc) + asyncio.futures.Future.cancel(t) + [cb[0](t) for cb in cbs] # cancels any dependent concurrent.futures + try: + t._coro.throw(exc) # exits coro, unless explicitly handled + except exc: + pass + return out + + +class AbstractAsyncStreamedFile(AbstractBufferedFile): + # no read buffering, and always auto-commit + # TODO: readahead might still be useful here, but needs async version + + async def read(self, length=-1): + """ + Return data from cache, or fetch pieces as necessary + + Parameters + ---------- + length: int (-1) + Number of bytes to read; if <0, all remaining bytes. + """ + length = -1 if length is None else int(length) + if self.mode != "rb": + raise ValueError("File not in read mode") + if length < 0: + length = self.size - self.loc + if self.closed: + raise ValueError("I/O operation on closed file.") + if length == 0: + # don't even bother calling fetch + return b"" + out = await self._fetch_range(self.loc, self.loc + length) + self.loc += len(out) + return out + + async def write(self, data): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be written. + """ + if self.mode not in {"wb", "ab"}: + raise ValueError("File not in write mode") + if self.closed: + raise ValueError("I/O operation on closed file.") + if self.forced: + raise ValueError("This file has been force-flushed, can only close") + out = self.buffer.write(data) + self.loc += out + if self.buffer.tell() >= self.blocksize: + await self.flush() + return out + + async def close(self): + """Close file + + Finalizes writes, discards cache + """ + if getattr(self, "_unclosable", False): + return + if self.closed: + return + if self.mode == "rb": + self.cache = None + else: + if not self.forced: + await self.flush(force=True) + + if self.fs is not None: + self.fs.invalidate_cache(self.path) + self.fs.invalidate_cache(self.fs._parent(self.path)) + + self.closed = True + + async def flush(self, force=False): + if self.closed: + raise ValueError("Flush on closed file") + if force and self.forced: + raise ValueError("Force flush cannot be called more than once") + if force: + self.forced = True + + if self.mode not in {"wb", "ab"}: + # no-op to flush on read-mode + return + + if not force and self.buffer.tell() < self.blocksize: + # Defer write on small block + return + + if self.offset is None: + # Initialize a multipart upload + self.offset = 0 + try: + await self._initiate_upload() + except: + self.closed = True + raise + + if await self._upload_chunk(final=force) is not False: + self.offset += self.buffer.seek(0, 2) + self.buffer = io.BytesIO() + + async def __aenter__(self): + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb): + await self.close() + + async def _fetch_range(self, start, end): + raise NotImplementedError + + async def _initiate_upload(self): + pass + + async def _upload_chunk(self, final=False): + raise NotImplementedError diff --git a/lib/python3.12/site-packages/fsspec/caching.py b/lib/python3.12/site-packages/fsspec/caching.py new file mode 100644 index 0000000000000000000000000000000000000000..34231c881e2b09bdc1c04e3969f1d68405a7af02 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/caching.py @@ -0,0 +1,966 @@ +from __future__ import annotations + +import collections +import functools +import logging +import math +import os +import threading +import warnings +from concurrent.futures import Future, ThreadPoolExecutor +from itertools import groupby +from operator import itemgetter +from typing import ( + TYPE_CHECKING, + Any, + Callable, + ClassVar, + Generic, + NamedTuple, + Optional, + OrderedDict, + TypeVar, +) + +if TYPE_CHECKING: + import mmap + + from typing_extensions import ParamSpec + + P = ParamSpec("P") +else: + P = TypeVar("P") + +T = TypeVar("T") + + +logger = logging.getLogger("fsspec") + +Fetcher = Callable[[int, int], bytes] # Maps (start, end) to bytes + + +class BaseCache: + """Pass-though cache: doesn't keep anything, calls every time + + Acts as base class for other cachers + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: func + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + """ + + name: ClassVar[str] = "none" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + self.blocksize = blocksize + self.nblocks = 0 + self.fetcher = fetcher + self.size = size + self.hit_count = 0 + self.miss_count = 0 + # the bytes that we actually requested + self.total_requested_bytes = 0 + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + if start is None: + start = 0 + if stop is None: + stop = self.size + if start >= self.size or start >= stop: + return b"" + return self.fetcher(start, stop) + + def _reset_stats(self) -> None: + """Reset hit and miss counts for a more ganular report e.g. by file.""" + self.hit_count = 0 + self.miss_count = 0 + self.total_requested_bytes = 0 + + def _log_stats(self) -> str: + """Return a formatted string of the cache statistics.""" + if self.hit_count == 0 and self.miss_count == 0: + # a cache that does nothing, this is for logs only + return "" + return f" , {self.name}: {self.hit_count} hits, {self.miss_count} misses, {self.total_requested_bytes} total requested bytes" + + def __repr__(self) -> str: + # TODO: use rich for better formatting + return f""" + <{self.__class__.__name__}: + block size : {self.blocksize} + block count : {self.nblocks} + file size : {self.size} + cache hits : {self.hit_count} + cache misses: {self.miss_count} + total requested bytes: {self.total_requested_bytes}> + """ + + +class MMapCache(BaseCache): + """memory-mapped sparse file cache + + Opens temporary file, which is filled blocks-wise when data is requested. + Ensure there is enough disc space in the temporary location. + + This cache method might only work on posix + """ + + name = "mmap" + + def __init__( + self, + blocksize: int, + fetcher: Fetcher, + size: int, + location: str | None = None, + blocks: set[int] | None = None, + ) -> None: + super().__init__(blocksize, fetcher, size) + self.blocks = set() if blocks is None else blocks + self.location = location + self.cache = self._makefile() + + def _makefile(self) -> mmap.mmap | bytearray: + import mmap + import tempfile + + if self.size == 0: + return bytearray() + + # posix version + if self.location is None or not os.path.exists(self.location): + if self.location is None: + fd = tempfile.TemporaryFile() + self.blocks = set() + else: + fd = open(self.location, "wb+") + fd.seek(self.size - 1) + fd.write(b"1") + fd.flush() + else: + fd = open(self.location, "r+b") + + return mmap.mmap(fd.fileno(), self.size) + + def _fetch(self, start: int | None, end: int | None) -> bytes: + logger.debug(f"MMap cache fetching {start}-{end}") + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + start_block = start // self.blocksize + end_block = end // self.blocksize + block_range = range(start_block, end_block + 1) + # Determine which blocks need to be fetched. This sequence is sorted by construction. + need = (i for i in block_range if i not in self.blocks) + # Count the number of blocks already cached + self.hit_count += sum(1 for i in block_range if i in self.blocks) + + # Consolidate needed blocks. + # Algorithm adapted from Python 2.x itertools documentation. + # We are grouping an enumerated sequence of blocks. By comparing when the difference + # between an ascending range (provided by enumerate) and the needed block numbers + # we can detect when the block number skips values. The key computes this difference. + # Whenever the difference changes, we know that we have previously cached block(s), + # and a new group is started. In other words, this algorithm neatly groups + # runs of consecutive block numbers so they can be fetched together. + for _, _blocks in groupby(enumerate(need), key=lambda x: x[0] - x[1]): + # Extract the blocks from the enumerated sequence + _blocks = tuple(map(itemgetter(1), _blocks)) + # Compute start of first block + sstart = _blocks[0] * self.blocksize + # Compute the end of the last block. Last block may not be full size. + send = min(_blocks[-1] * self.blocksize + self.blocksize, self.size) + + # Fetch bytes (could be multiple consecutive blocks) + self.total_requested_bytes += send - sstart + logger.debug( + f"MMap get blocks {_blocks[0]}-{_blocks[-1]} ({sstart}-{send})" + ) + self.cache[sstart:send] = self.fetcher(sstart, send) + + # Update set of cached blocks + self.blocks.update(_blocks) + # Update cache statistics with number of blocks we had to cache + self.miss_count += len(_blocks) + + return self.cache[start:end] + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__.copy() + # Remove the unpicklable entries. + del state["cache"] + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + # Restore instance attributes + self.__dict__.update(state) + self.cache = self._makefile() + + +class ReadAheadCache(BaseCache): + """Cache which reads only when we get beyond a block of data + + This is a much simpler version of BytesCache, and does not attempt to + fill holes in the cache or keep fragments alive. It is best suited to + many small reads in a sequential order (e.g., reading lines from a file). + """ + + name = "readahead" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + super().__init__(blocksize, fetcher, size) + self.cache = b"" + self.start = 0 + self.end = 0 + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None or end > self.size: + end = self.size + if start >= self.size or start >= end: + return b"" + l = end - start + if start >= self.start and end <= self.end: + # cache hit + self.hit_count += 1 + return self.cache[start - self.start : end - self.start] + elif self.start <= start < self.end: + # partial hit + self.miss_count += 1 + part = self.cache[start - self.start :] + l -= len(part) + start = self.end + else: + # miss + self.miss_count += 1 + part = b"" + end = min(self.size, end + self.blocksize) + self.total_requested_bytes += end - start + self.cache = self.fetcher(start, end) # new block replaces old + self.start = start + self.end = self.start + len(self.cache) + return part + self.cache[:l] + + +class FirstChunkCache(BaseCache): + """Caches the first block of a file only + + This may be useful for file types where the metadata is stored in the header, + but is randomly accessed. + """ + + name = "first" + + def __init__(self, blocksize: int, fetcher: Fetcher, size: int) -> None: + if blocksize > size: + # this will buffer the whole thing + blocksize = size + super().__init__(blocksize, fetcher, size) + self.cache: bytes | None = None + + def _fetch(self, start: int | None, end: int | None) -> bytes: + start = start or 0 + if start > self.size: + logger.debug("FirstChunkCache: requested start > file size") + return b"" + + end = min(end, self.size) + + if start < self.blocksize: + if self.cache is None: + self.miss_count += 1 + if end > self.blocksize: + self.total_requested_bytes += end + data = self.fetcher(0, end) + self.cache = data[: self.blocksize] + return data[start:] + self.cache = self.fetcher(0, self.blocksize) + self.total_requested_bytes += self.blocksize + part = self.cache[start:end] + if end > self.blocksize: + self.total_requested_bytes += end - self.blocksize + part += self.fetcher(self.blocksize, end) + self.hit_count += 1 + return part + else: + self.miss_count += 1 + self.total_requested_bytes += end - start + return self.fetcher(start, end) + + +class BlockCache(BaseCache): + """ + Cache holding memory as a set of blocks. + + Requests are only ever made ``blocksize`` at a time, and are + stored in an LRU cache. The least recently accessed block is + discarded when more than ``maxblocks`` are stored. + + Parameters + ---------- + blocksize : int + The number of bytes to store in each block. + Requests are only ever made for ``blocksize``, so this + should balance the overhead of making a request against + the granularity of the blocks. + fetcher : Callable + size : int + The total size of the file being cached. + maxblocks : int + The maximum number of blocks to cache for. The maximum memory + use for this cache is then ``blocksize * maxblocks``. + """ + + name = "blockcache" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 + ) -> None: + super().__init__(blocksize, fetcher, size) + self.nblocks = math.ceil(size / blocksize) + self.maxblocks = maxblocks + self._fetch_block_cached = functools.lru_cache(maxblocks)(self._fetch_block) + + def cache_info(self): + """ + The statistics on the block cache. + + Returns + ------- + NamedTuple + Returned directly from the LRU Cache used internally. + """ + return self._fetch_block_cached.cache_info() + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__ + del state["_fetch_block_cached"] + return state + + def __setstate__(self, state: dict[str, Any]) -> None: + self.__dict__.update(state) + self._fetch_block_cached = functools.lru_cache(state["maxblocks"])( + self._fetch_block + ) + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + + # byte position -> block numbers + start_block_number = start // self.blocksize + end_block_number = end // self.blocksize + + # these are cached, so safe to do multiple calls for the same start and end. + for block_number in range(start_block_number, end_block_number + 1): + self._fetch_block_cached(block_number) + + return self._read_cache( + start, + end, + start_block_number=start_block_number, + end_block_number=end_block_number, + ) + + def _fetch_block(self, block_number: int) -> bytes: + """ + Fetch the block of data for `block_number`. + """ + if block_number > self.nblocks: + raise ValueError( + f"'block_number={block_number}' is greater than " + f"the number of blocks ({self.nblocks})" + ) + + start = block_number * self.blocksize + end = start + self.blocksize + self.total_requested_bytes += end - start + self.miss_count += 1 + logger.info("BlockCache fetching block %d", block_number) + block_contents = super()._fetch(start, end) + return block_contents + + def _read_cache( + self, start: int, end: int, start_block_number: int, end_block_number: int + ) -> bytes: + """ + Read from our block cache. + + Parameters + ---------- + start, end : int + The start and end byte positions. + start_block_number, end_block_number : int + The start and end block numbers. + """ + start_pos = start % self.blocksize + end_pos = end % self.blocksize + + self.hit_count += 1 + if start_block_number == end_block_number: + block: bytes = self._fetch_block_cached(start_block_number) + return block[start_pos:end_pos] + + else: + # read from the initial + out = [self._fetch_block_cached(start_block_number)[start_pos:]] + + # intermediate blocks + # Note: it'd be nice to combine these into one big request. However + # that doesn't play nicely with our LRU cache. + out.extend( + map( + self._fetch_block_cached, + range(start_block_number + 1, end_block_number), + ) + ) + + # final block + out.append(self._fetch_block_cached(end_block_number)[:end_pos]) + + return b"".join(out) + + +class BytesCache(BaseCache): + """Cache which holds data in a in-memory bytes object + + Implements read-ahead by the block size, for semi-random reads progressing + through the file. + + Parameters + ---------- + trim: bool + As we read more data, whether to discard the start of the buffer when + we are more than a blocksize ahead of it. + """ + + name: ClassVar[str] = "bytes" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, trim: bool = True + ) -> None: + super().__init__(blocksize, fetcher, size) + self.cache = b"" + self.start: int | None = None + self.end: int | None = None + self.trim = trim + + def _fetch(self, start: int | None, end: int | None) -> bytes: + # TODO: only set start/end after fetch, in case it fails? + # is this where retry logic might go? + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + if ( + self.start is not None + and start >= self.start + and self.end is not None + and end < self.end + ): + # cache hit: we have all the required data + offset = start - self.start + self.hit_count += 1 + return self.cache[offset : offset + end - start] + + if self.blocksize: + bend = min(self.size, end + self.blocksize) + else: + bend = end + + if bend == start or start > self.size: + return b"" + + if (self.start is None or start < self.start) and ( + self.end is None or end > self.end + ): + # First read, or extending both before and after + self.total_requested_bytes += bend - start + self.miss_count += 1 + self.cache = self.fetcher(start, bend) + self.start = start + else: + assert self.start is not None + assert self.end is not None + self.miss_count += 1 + + if start < self.start: + if self.end is None or self.end - end > self.blocksize: + self.total_requested_bytes += bend - start + self.cache = self.fetcher(start, bend) + self.start = start + else: + self.total_requested_bytes += self.start - start + new = self.fetcher(start, self.start) + self.start = start + self.cache = new + self.cache + elif self.end is not None and bend > self.end: + if self.end > self.size: + pass + elif end - self.end > self.blocksize: + self.total_requested_bytes += bend - start + self.cache = self.fetcher(start, bend) + self.start = start + else: + self.total_requested_bytes += bend - self.end + new = self.fetcher(self.end, bend) + self.cache = self.cache + new + + self.end = self.start + len(self.cache) + offset = start - self.start + out = self.cache[offset : offset + end - start] + if self.trim: + num = (self.end - self.start) // (self.blocksize + 1) + if num > 1: + self.start += self.blocksize * num + self.cache = self.cache[self.blocksize * num :] + return out + + def __len__(self) -> int: + return len(self.cache) + + +class AllBytes(BaseCache): + """Cache entire contents of the file""" + + name: ClassVar[str] = "all" + + def __init__( + self, + blocksize: int | None = None, + fetcher: Fetcher | None = None, + size: int | None = None, + data: bytes | None = None, + ) -> None: + super().__init__(blocksize, fetcher, size) # type: ignore[arg-type] + if data is None: + self.miss_count += 1 + self.total_requested_bytes += self.size + data = self.fetcher(0, self.size) + self.data = data + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + self.hit_count += 1 + return self.data[start:stop] + + +class KnownPartsOfAFile(BaseCache): + """ + Cache holding known file parts. + + Parameters + ---------- + blocksize: int + How far to read ahead in numbers of bytes + fetcher: func + Function of the form f(start, end) which gets bytes from remote as + specified + size: int + How big this file is + data: dict + A dictionary mapping explicit `(start, stop)` file-offset tuples + with known bytes. + strict: bool, default True + Whether to fetch reads that go beyond a known byte-range boundary. + If `False`, any read that ends outside a known part will be zero + padded. Note that zero padding will not be used for reads that + begin outside a known byte-range. + """ + + name: ClassVar[str] = "parts" + + def __init__( + self, + blocksize: int, + fetcher: Fetcher, + size: int, + data: Optional[dict[tuple[int, int], bytes]] = None, + strict: bool = True, + **_: Any, + ): + super().__init__(blocksize, fetcher, size) + self.strict = strict + + # simple consolidation of contiguous blocks + if data: + old_offsets = sorted(data.keys()) + offsets = [old_offsets[0]] + blocks = [data.pop(old_offsets[0])] + for start, stop in old_offsets[1:]: + start0, stop0 = offsets[-1] + if start == stop0: + offsets[-1] = (start0, stop) + blocks[-1] += data.pop((start, stop)) + else: + offsets.append((start, stop)) + blocks.append(data.pop((start, stop))) + + self.data = dict(zip(offsets, blocks)) + else: + self.data = {} + + def _fetch(self, start: int | None, stop: int | None) -> bytes: + if start is None: + start = 0 + if stop is None: + stop = self.size + + out = b"" + for (loc0, loc1), data in self.data.items(): + # If self.strict=False, use zero-padded data + # for reads beyond the end of a "known" buffer + if loc0 <= start < loc1: + off = start - loc0 + out = data[off : off + stop - start] + if not self.strict or loc0 <= stop <= loc1: + # The request is within a known range, or + # it begins within a known range, and we + # are allowed to pad reads beyond the + # buffer with zero + out += b"\x00" * (stop - start - len(out)) + self.hit_count += 1 + return out + else: + # The request ends outside a known range, + # and we are being "strict" about reads + # beyond the buffer + start = loc1 + break + + # We only get here if there is a request outside the + # known parts of the file. In an ideal world, this + # should never happen + if self.fetcher is None: + # We cannot fetch the data, so raise an error + raise ValueError(f"Read is outside the known file parts: {(start, stop)}. ") + # We can fetch the data, but should warn the user + # that this may be slow + warnings.warn( + f"Read is outside the known file parts: {(start, stop)}. " + f"IO/caching performance may be poor!" + ) + logger.debug(f"KnownPartsOfAFile cache fetching {start}-{stop}") + self.total_requested_bytes += stop - start + self.miss_count += 1 + return out + super()._fetch(start, stop) + + +class UpdatableLRU(Generic[P, T]): + """ + Custom implementation of LRU cache that allows updating keys + + Used by BackgroudBlockCache + """ + + class CacheInfo(NamedTuple): + hits: int + misses: int + maxsize: int + currsize: int + + def __init__(self, func: Callable[P, T], max_size: int = 128) -> None: + self._cache: OrderedDict[Any, T] = collections.OrderedDict() + self._func = func + self._max_size = max_size + self._hits = 0 + self._misses = 0 + self._lock = threading.Lock() + + def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: + if kwargs: + raise TypeError(f"Got unexpected keyword argument {kwargs.keys()}") + with self._lock: + if args in self._cache: + self._cache.move_to_end(args) + self._hits += 1 + return self._cache[args] + + result = self._func(*args, **kwargs) + + with self._lock: + self._cache[args] = result + self._misses += 1 + if len(self._cache) > self._max_size: + self._cache.popitem(last=False) + + return result + + def is_key_cached(self, *args: Any) -> bool: + with self._lock: + return args in self._cache + + def add_key(self, result: T, *args: Any) -> None: + with self._lock: + self._cache[args] = result + if len(self._cache) > self._max_size: + self._cache.popitem(last=False) + + def cache_info(self) -> UpdatableLRU.CacheInfo: + with self._lock: + return self.CacheInfo( + maxsize=self._max_size, + currsize=len(self._cache), + hits=self._hits, + misses=self._misses, + ) + + +class BackgroundBlockCache(BaseCache): + """ + Cache holding memory as a set of blocks with pre-loading of + the next block in the background. + + Requests are only ever made ``blocksize`` at a time, and are + stored in an LRU cache. The least recently accessed block is + discarded when more than ``maxblocks`` are stored. If the + next block is not in cache, it is loaded in a separate thread + in non-blocking way. + + Parameters + ---------- + blocksize : int + The number of bytes to store in each block. + Requests are only ever made for ``blocksize``, so this + should balance the overhead of making a request against + the granularity of the blocks. + fetcher : Callable + size : int + The total size of the file being cached. + maxblocks : int + The maximum number of blocks to cache for. The maximum memory + use for this cache is then ``blocksize * maxblocks``. + """ + + name: ClassVar[str] = "background" + + def __init__( + self, blocksize: int, fetcher: Fetcher, size: int, maxblocks: int = 32 + ) -> None: + super().__init__(blocksize, fetcher, size) + self.nblocks = math.ceil(size / blocksize) + self.maxblocks = maxblocks + self._fetch_block_cached = UpdatableLRU(self._fetch_block, maxblocks) + + self._thread_executor = ThreadPoolExecutor(max_workers=1) + self._fetch_future_block_number: int | None = None + self._fetch_future: Future[bytes] | None = None + self._fetch_future_lock = threading.Lock() + + def cache_info(self) -> UpdatableLRU.CacheInfo: + """ + The statistics on the block cache. + + Returns + ------- + NamedTuple + Returned directly from the LRU Cache used internally. + """ + return self._fetch_block_cached.cache_info() + + def __getstate__(self) -> dict[str, Any]: + state = self.__dict__ + del state["_fetch_block_cached"] + del state["_thread_executor"] + del state["_fetch_future_block_number"] + del state["_fetch_future"] + del state["_fetch_future_lock"] + return state + + def __setstate__(self, state) -> None: + self.__dict__.update(state) + self._fetch_block_cached = UpdatableLRU(self._fetch_block, state["maxblocks"]) + self._thread_executor = ThreadPoolExecutor(max_workers=1) + self._fetch_future_block_number = None + self._fetch_future = None + self._fetch_future_lock = threading.Lock() + + def _fetch(self, start: int | None, end: int | None) -> bytes: + if start is None: + start = 0 + if end is None: + end = self.size + if start >= self.size or start >= end: + return b"" + + # byte position -> block numbers + start_block_number = start // self.blocksize + end_block_number = end // self.blocksize + + fetch_future_block_number = None + fetch_future = None + with self._fetch_future_lock: + # Background thread is running. Check we we can or must join it. + if self._fetch_future is not None: + assert self._fetch_future_block_number is not None + if self._fetch_future.done(): + logger.info("BlockCache joined background fetch without waiting.") + self._fetch_block_cached.add_key( + self._fetch_future.result(), self._fetch_future_block_number + ) + # Cleanup the fetch variables. Done with fetching the block. + self._fetch_future_block_number = None + self._fetch_future = None + else: + # Must join if we need the block for the current fetch + must_join = bool( + start_block_number + <= self._fetch_future_block_number + <= end_block_number + ) + if must_join: + # Copy to the local variables to release lock + # before waiting for result + fetch_future_block_number = self._fetch_future_block_number + fetch_future = self._fetch_future + + # Cleanup the fetch variables. Have a local copy. + self._fetch_future_block_number = None + self._fetch_future = None + + # Need to wait for the future for the current read + if fetch_future is not None: + logger.info("BlockCache waiting for background fetch.") + # Wait until result and put it in cache + self._fetch_block_cached.add_key( + fetch_future.result(), fetch_future_block_number + ) + + # these are cached, so safe to do multiple calls for the same start and end. + for block_number in range(start_block_number, end_block_number + 1): + self._fetch_block_cached(block_number) + + # fetch next block in the background if nothing is running in the background, + # the block is within file and it is not already cached + end_block_plus_1 = end_block_number + 1 + with self._fetch_future_lock: + if ( + self._fetch_future is None + and end_block_plus_1 <= self.nblocks + and not self._fetch_block_cached.is_key_cached(end_block_plus_1) + ): + self._fetch_future_block_number = end_block_plus_1 + self._fetch_future = self._thread_executor.submit( + self._fetch_block, end_block_plus_1, "async" + ) + + return self._read_cache( + start, + end, + start_block_number=start_block_number, + end_block_number=end_block_number, + ) + + def _fetch_block(self, block_number: int, log_info: str = "sync") -> bytes: + """ + Fetch the block of data for `block_number`. + """ + if block_number > self.nblocks: + raise ValueError( + f"'block_number={block_number}' is greater than " + f"the number of blocks ({self.nblocks})" + ) + + start = block_number * self.blocksize + end = start + self.blocksize + logger.info("BlockCache fetching block (%s) %d", log_info, block_number) + self.total_requested_bytes += end - start + self.miss_count += 1 + block_contents = super()._fetch(start, end) + return block_contents + + def _read_cache( + self, start: int, end: int, start_block_number: int, end_block_number: int + ) -> bytes: + """ + Read from our block cache. + + Parameters + ---------- + start, end : int + The start and end byte positions. + start_block_number, end_block_number : int + The start and end block numbers. + """ + start_pos = start % self.blocksize + end_pos = end % self.blocksize + + # kind of pointless to count this as a hit, but it is + self.hit_count += 1 + + if start_block_number == end_block_number: + block = self._fetch_block_cached(start_block_number) + return block[start_pos:end_pos] + + else: + # read from the initial + out = [self._fetch_block_cached(start_block_number)[start_pos:]] + + # intermediate blocks + # Note: it'd be nice to combine these into one big request. However + # that doesn't play nicely with our LRU cache. + out.extend( + map( + self._fetch_block_cached, + range(start_block_number + 1, end_block_number), + ) + ) + + # final block + out.append(self._fetch_block_cached(end_block_number)[:end_pos]) + + return b"".join(out) + + +caches: dict[str | None, type[BaseCache]] = { + # one custom case + None: BaseCache, +} + + +def register_cache(cls: type[BaseCache], clobber: bool = False) -> None: + """'Register' cache implementation. + + Parameters + ---------- + clobber: bool, optional + If set to True (default is False) - allow to overwrite existing + entry. + + Raises + ------ + ValueError + """ + name = cls.name + if not clobber and name in caches: + raise ValueError(f"Cache with name {name!r} is already known: {caches[name]}") + caches[name] = cls + + +for c in ( + BaseCache, + MMapCache, + BytesCache, + ReadAheadCache, + BlockCache, + FirstChunkCache, + AllBytes, + KnownPartsOfAFile, + BackgroundBlockCache, +): + register_cache(c) diff --git a/lib/python3.12/site-packages/fsspec/callbacks.py b/lib/python3.12/site-packages/fsspec/callbacks.py new file mode 100644 index 0000000000000000000000000000000000000000..7ca99ca6ac3cd69b28bcd1550f6550e8e648c5fe --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/callbacks.py @@ -0,0 +1,324 @@ +from functools import wraps + + +class Callback: + """ + Base class and interface for callback mechanism + + This class can be used directly for monitoring file transfers by + providing ``callback=Callback(hooks=...)`` (see the ``hooks`` argument, + below), or subclassed for more specialised behaviour. + + Parameters + ---------- + size: int (optional) + Nominal quantity for the value that corresponds to a complete + transfer, e.g., total number of tiles or total number of + bytes + value: int (0) + Starting internal counter value + hooks: dict or None + A dict of named functions to be called on each update. The signature + of these must be ``f(size, value, **kwargs)`` + """ + + def __init__(self, size=None, value=0, hooks=None, **kwargs): + self.size = size + self.value = value + self.hooks = hooks or {} + self.kw = kwargs + + def __enter__(self): + return self + + def __exit__(self, *exc_args): + self.close() + + def close(self): + """Close callback.""" + + def branched(self, path_1, path_2, **kwargs): + """ + Return callback for child transfers + + If this callback is operating at a higher level, e.g., put, which may + trigger transfers that can also be monitored. The function returns a callback + that has to be passed to the child method, e.g., put_file, + as `callback=` argument. + + The implementation uses `callback.branch` for compatibility. + When implementing callbacks, it is recommended to override this function instead + of `branch` and avoid calling `super().branched(...)`. + + Prefer using this function over `branch`. + + Parameters + ---------- + path_1: str + Child's source path + path_2: str + Child's destination path + **kwargs: + Arbitrary keyword arguments + + Returns + ------- + callback: Callback + A callback instance to be passed to the child method + """ + self.branch(path_1, path_2, kwargs) + # mutate kwargs so that we can force the caller to pass "callback=" explicitly + return kwargs.pop("callback", DEFAULT_CALLBACK) + + def branch_coro(self, fn): + """ + Wraps a coroutine, and pass a new child callback to it. + """ + + @wraps(fn) + async def func(path1, path2: str, **kwargs): + with self.branched(path1, path2, **kwargs) as child: + return await fn(path1, path2, callback=child, **kwargs) + + return func + + def set_size(self, size): + """ + Set the internal maximum size attribute + + Usually called if not initially set at instantiation. Note that this + triggers a ``call()``. + + Parameters + ---------- + size: int + """ + self.size = size + self.call() + + def absolute_update(self, value): + """ + Set the internal value state + + Triggers ``call()`` + + Parameters + ---------- + value: int + """ + self.value = value + self.call() + + def relative_update(self, inc=1): + """ + Delta increment the internal counter + + Triggers ``call()`` + + Parameters + ---------- + inc: int + """ + self.value += inc + self.call() + + def call(self, hook_name=None, **kwargs): + """ + Execute hook(s) with current state + + Each function is passed the internal size and current value + + Parameters + ---------- + hook_name: str or None + If given, execute on this hook + kwargs: passed on to (all) hook(s) + """ + if not self.hooks: + return + kw = self.kw.copy() + kw.update(kwargs) + if hook_name: + if hook_name not in self.hooks: + return + return self.hooks[hook_name](self.size, self.value, **kw) + for hook in self.hooks.values() or []: + hook(self.size, self.value, **kw) + + def wrap(self, iterable): + """ + Wrap an iterable to call ``relative_update`` on each iterations + + Parameters + ---------- + iterable: Iterable + The iterable that is being wrapped + """ + for item in iterable: + self.relative_update() + yield item + + def branch(self, path_1, path_2, kwargs): + """ + Set callbacks for child transfers + + If this callback is operating at a higher level, e.g., put, which may + trigger transfers that can also be monitored. The passed kwargs are + to be *mutated* to add ``callback=``, if this class supports branching + to children. + + Parameters + ---------- + path_1: str + Child's source path + path_2: str + Child's destination path + kwargs: dict + arguments passed to child method, e.g., put_file. + + Returns + ------- + + """ + return None + + def no_op(self, *_, **__): + pass + + def __getattr__(self, item): + """ + If undefined methods are called on this class, nothing happens + """ + return self.no_op + + @classmethod + def as_callback(cls, maybe_callback=None): + """Transform callback=... into Callback instance + + For the special value of ``None``, return the global instance of + ``NoOpCallback``. This is an alternative to including + ``callback=DEFAULT_CALLBACK`` directly in a method signature. + """ + if maybe_callback is None: + return DEFAULT_CALLBACK + return maybe_callback + + +class NoOpCallback(Callback): + """ + This implementation of Callback does exactly nothing + """ + + def call(self, *args, **kwargs): + return None + + +class DotPrinterCallback(Callback): + """ + Simple example Callback implementation + + Almost identical to Callback with a hook that prints a char; here we + demonstrate how the outer layer may print "#" and the inner layer "." + """ + + def __init__(self, chr_to_print="#", **kwargs): + self.chr = chr_to_print + super().__init__(**kwargs) + + def branch(self, path_1, path_2, kwargs): + """Mutate kwargs to add new instance with different print char""" + kwargs["callback"] = DotPrinterCallback(".") + + def call(self, **kwargs): + """Just outputs a character""" + print(self.chr, end="") + + +class TqdmCallback(Callback): + """ + A callback to display a progress bar using tqdm + + Parameters + ---------- + tqdm_kwargs : dict, (optional) + Any argument accepted by the tqdm constructor. + See the `tqdm doc `_. + Will be forwarded to `tqdm_cls`. + tqdm_cls: (optional) + subclass of `tqdm.tqdm`. If not passed, it will default to `tqdm.tqdm`. + + Examples + -------- + >>> import fsspec + >>> from fsspec.callbacks import TqdmCallback + >>> fs = fsspec.filesystem("memory") + >>> path2distant_data = "/your-path" + >>> fs.upload( + ".", + path2distant_data, + recursive=True, + callback=TqdmCallback(), + ) + + You can forward args to tqdm using the ``tqdm_kwargs`` parameter. + + >>> fs.upload( + ".", + path2distant_data, + recursive=True, + callback=TqdmCallback(tqdm_kwargs={"desc": "Your tqdm description"}), + ) + + You can also customize the progress bar by passing a subclass of `tqdm`. + + .. code-block:: python + + class TqdmFormat(tqdm): + '''Provides a `total_time` format parameter''' + @property + def format_dict(self): + d = super().format_dict + total_time = d["elapsed"] * (d["total"] or 0) / max(d["n"], 1) + d.update(total_time=self.format_interval(total_time) + " in total") + return d + + >>> with TqdmCallback( + tqdm_kwargs={ + "desc": "desc", + "bar_format": "{total_time}: {percentage:.0f}%|{bar}{r_bar}", + }, + tqdm_cls=TqdmFormat, + ) as callback: + fs.upload(".", path2distant_data, recursive=True, callback=callback) + """ + + def __init__(self, tqdm_kwargs=None, *args, **kwargs): + try: + from tqdm import tqdm + + except ImportError as exce: + raise ImportError( + "Using TqdmCallback requires tqdm to be installed" + ) from exce + + self._tqdm_cls = kwargs.pop("tqdm_cls", tqdm) + self._tqdm_kwargs = tqdm_kwargs or {} + self.tqdm = None + super().__init__(*args, **kwargs) + + def call(self, *args, **kwargs): + if self.tqdm is None: + self.tqdm = self._tqdm_cls(total=self.size, **self._tqdm_kwargs) + self.tqdm.total = self.size + self.tqdm.update(self.value - self.tqdm.n) + + def close(self): + if self.tqdm is not None: + self.tqdm.close() + self.tqdm = None + + def __del__(self): + return self.close() + + +DEFAULT_CALLBACK = _DEFAULT_CALLBACK = NoOpCallback() diff --git a/lib/python3.12/site-packages/fsspec/compression.py b/lib/python3.12/site-packages/fsspec/compression.py new file mode 100644 index 0000000000000000000000000000000000000000..fc519c24532fdcd3f6f1b3fc645d38e1e1dde1d7 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/compression.py @@ -0,0 +1,175 @@ +"""Helper functions for a standard streaming compression API""" + +from zipfile import ZipFile + +import fsspec.utils +from fsspec.spec import AbstractBufferedFile + + +def noop_file(file, mode, **kwargs): + return file + + +# TODO: files should also be available as contexts +# should be functions of the form func(infile, mode=, **kwargs) -> file-like +compr = {None: noop_file} + + +def register_compression(name, callback, extensions, force=False): + """Register an "inferable" file compression type. + + Registers transparent file compression type for use with fsspec.open. + Compression can be specified by name in open, or "infer"-ed for any files + ending with the given extensions. + + Args: + name: (str) The compression type name. Eg. "gzip". + callback: A callable of form (infile, mode, **kwargs) -> file-like. + Accepts an input file-like object, the target mode and kwargs. + Returns a wrapped file-like object. + extensions: (str, Iterable[str]) A file extension, or list of file + extensions for which to infer this compression scheme. Eg. "gz". + force: (bool) Force re-registration of compression type or extensions. + + Raises: + ValueError: If name or extensions already registered, and not force. + + """ + if isinstance(extensions, str): + extensions = [extensions] + + # Validate registration + if name in compr and not force: + raise ValueError(f"Duplicate compression registration: {name}") + + for ext in extensions: + if ext in fsspec.utils.compressions and not force: + raise ValueError(f"Duplicate compression file extension: {ext} ({name})") + + compr[name] = callback + + for ext in extensions: + fsspec.utils.compressions[ext] = name + + +def unzip(infile, mode="rb", filename=None, **kwargs): + if "r" not in mode: + filename = filename or "file" + z = ZipFile(infile, mode="w", **kwargs) + fo = z.open(filename, mode="w") + fo.close = lambda closer=fo.close: closer() or z.close() + return fo + z = ZipFile(infile) + if filename is None: + filename = z.namelist()[0] + return z.open(filename, mode="r", **kwargs) + + +register_compression("zip", unzip, "zip") + +try: + from bz2 import BZ2File +except ImportError: + pass +else: + register_compression("bz2", BZ2File, "bz2") + +try: # pragma: no cover + from isal import igzip + + def isal(infile, mode="rb", **kwargs): + return igzip.IGzipFile(fileobj=infile, mode=mode, **kwargs) + + register_compression("gzip", isal, "gz") +except ImportError: + from gzip import GzipFile + + register_compression( + "gzip", lambda f, **kwargs: GzipFile(fileobj=f, **kwargs), "gz" + ) + +try: + from lzma import LZMAFile + + register_compression("lzma", LZMAFile, "lzma") + register_compression("xz", LZMAFile, "xz") +except ImportError: + pass + +try: + import lzmaffi + + register_compression("lzma", lzmaffi.LZMAFile, "lzma", force=True) + register_compression("xz", lzmaffi.LZMAFile, "xz", force=True) +except ImportError: + pass + + +class SnappyFile(AbstractBufferedFile): + def __init__(self, infile, mode, **kwargs): + import snappy + + super().__init__( + fs=None, path="snappy", mode=mode.strip("b") + "b", size=999999999, **kwargs + ) + self.infile = infile + if "r" in mode: + self.codec = snappy.StreamDecompressor() + else: + self.codec = snappy.StreamCompressor() + + def _upload_chunk(self, final=False): + self.buffer.seek(0) + out = self.codec.add_chunk(self.buffer.read()) + self.infile.write(out) + return True + + def seek(self, loc, whence=0): + raise NotImplementedError("SnappyFile is not seekable") + + def seekable(self): + return False + + def _fetch_range(self, start, end): + """Get the specified set of bytes from remote""" + data = self.infile.read(end - start) + return self.codec.decompress(data) + + +try: + import snappy + + snappy.compress(b"") + # Snappy may use the .sz file extension, but this is not part of the + # standard implementation. + register_compression("snappy", SnappyFile, []) + +except (ImportError, NameError, AttributeError): + pass + +try: + import lz4.frame + + register_compression("lz4", lz4.frame.open, "lz4") +except ImportError: + pass + +try: + import zstandard as zstd + + def zstandard_file(infile, mode="rb"): + if "r" in mode: + cctx = zstd.ZstdDecompressor() + return cctx.stream_reader(infile) + else: + cctx = zstd.ZstdCompressor(level=10) + return cctx.stream_writer(infile) + + register_compression("zstd", zstandard_file, "zst") +except ImportError: + pass + + +def available_compressions(): + """Return a list of the implemented compressions.""" + return list(compr) diff --git a/lib/python3.12/site-packages/fsspec/config.py b/lib/python3.12/site-packages/fsspec/config.py new file mode 100644 index 0000000000000000000000000000000000000000..76d9af14aaf7df47c4551c169f27b05abf9c269e --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/config.py @@ -0,0 +1,131 @@ +from __future__ import annotations + +import configparser +import json +import os +import warnings +from typing import Any + +conf: dict[str, dict[str, Any]] = {} +default_conf_dir = os.path.join(os.path.expanduser("~"), ".config/fsspec") +conf_dir = os.environ.get("FSSPEC_CONFIG_DIR", default_conf_dir) + + +def set_conf_env(conf_dict, envdict=os.environ): + """Set config values from environment variables + + Looks for variables of the form ``FSSPEC_`` and + ``FSSPEC__``. For ``FSSPEC_`` the value is parsed + as a json dictionary and used to ``update`` the config of the + corresponding protocol. For ``FSSPEC__`` there is no + attempt to convert the string value, but the kwarg keys will be lower-cased. + + The ``FSSPEC__`` variables are applied after the + ``FSSPEC_`` ones. + + Parameters + ---------- + conf_dict : dict(str, dict) + This dict will be mutated + envdict : dict-like(str, str) + Source for the values - usually the real environment + """ + kwarg_keys = [] + for key in envdict: + if key.startswith("FSSPEC_") and len(key) > 7 and key[7] != "_": + if key.count("_") > 1: + kwarg_keys.append(key) + continue + try: + value = json.loads(envdict[key]) + except json.decoder.JSONDecodeError as ex: + warnings.warn( + f"Ignoring environment variable {key} due to a parse failure: {ex}" + ) + else: + if isinstance(value, dict): + _, proto = key.split("_", 1) + conf_dict.setdefault(proto.lower(), {}).update(value) + else: + warnings.warn( + f"Ignoring environment variable {key} due to not being a dict:" + f" {type(value)}" + ) + elif key.startswith("FSSPEC"): + warnings.warn( + f"Ignoring environment variable {key} due to having an unexpected name" + ) + + for key in kwarg_keys: + _, proto, kwarg = key.split("_", 2) + conf_dict.setdefault(proto.lower(), {})[kwarg.lower()] = envdict[key] + + +def set_conf_files(cdir, conf_dict): + """Set config values from files + + Scans for INI and JSON files in the given dictionary, and uses their + contents to set the config. In case of repeated values, later values + win. + + In the case of INI files, all values are strings, and these will not + be converted. + + Parameters + ---------- + cdir : str + Directory to search + conf_dict : dict(str, dict) + This dict will be mutated + """ + if not os.path.isdir(cdir): + return + allfiles = sorted(os.listdir(cdir)) + for fn in allfiles: + if fn.endswith(".ini"): + ini = configparser.ConfigParser() + ini.read(os.path.join(cdir, fn)) + for key in ini: + if key == "DEFAULT": + continue + conf_dict.setdefault(key, {}).update(dict(ini[key])) + if fn.endswith(".json"): + with open(os.path.join(cdir, fn)) as f: + js = json.load(f) + for key in js: + conf_dict.setdefault(key, {}).update(dict(js[key])) + + +def apply_config(cls, kwargs, conf_dict=None): + """Supply default values for kwargs when instantiating class + + Augments the passed kwargs, by finding entries in the config dict + which match the classes ``.protocol`` attribute (one or more str) + + Parameters + ---------- + cls : file system implementation + kwargs : dict + conf_dict : dict of dict + Typically this is the global configuration + + Returns + ------- + dict : the modified set of kwargs + """ + if conf_dict is None: + conf_dict = conf + protos = cls.protocol if isinstance(cls.protocol, (tuple, list)) else [cls.protocol] + kw = {} + for proto in protos: + # default kwargs from the current state of the config + if proto in conf_dict: + kw.update(conf_dict[proto]) + # explicit kwargs always win + kw.update(**kwargs) + kwargs = kw + return kwargs + + +set_conf_files(conf_dir, conf) +set_conf_env(conf) diff --git a/lib/python3.12/site-packages/fsspec/conftest.py b/lib/python3.12/site-packages/fsspec/conftest.py new file mode 100644 index 0000000000000000000000000000000000000000..6874a42c4895c3c7b973dc5d63fd4488a4e60b44 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/conftest.py @@ -0,0 +1,55 @@ +import os +import shutil +import subprocess +import sys +import time + +import pytest + +import fsspec +from fsspec.implementations.cached import CachingFileSystem + + +@pytest.fixture() +def m(): + """ + Fixture providing a memory filesystem. + """ + m = fsspec.filesystem("memory") + m.store.clear() + m.pseudo_dirs.clear() + m.pseudo_dirs.append("") + try: + yield m + finally: + m.store.clear() + m.pseudo_dirs.clear() + m.pseudo_dirs.append("") + + +@pytest.fixture +def ftp_writable(tmpdir): + """ + Fixture providing a writable FTP filesystem. + """ + pytest.importorskip("pyftpdlib") + from fsspec.implementations.ftp import FTPFileSystem + + FTPFileSystem.clear_instance_cache() # remove lingering connections + CachingFileSystem.clear_instance_cache() + d = str(tmpdir) + with open(os.path.join(d, "out"), "wb") as f: + f.write(b"hello" * 10000) + P = subprocess.Popen( + [sys.executable, "-m", "pyftpdlib", "-d", d, "-u", "user", "-P", "pass", "-w"] + ) + try: + time.sleep(1) + yield "localhost", 2121, "user", "pass" + finally: + P.terminate() + P.wait() + try: + shutil.rmtree(tmpdir) + except Exception: + pass diff --git a/lib/python3.12/site-packages/fsspec/core.py b/lib/python3.12/site-packages/fsspec/core.py new file mode 100644 index 0000000000000000000000000000000000000000..d8e75572bc0e31b5a12faee73275760e421aadb0 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/core.py @@ -0,0 +1,743 @@ +from __future__ import annotations + +import io +import logging +import os +import re +from glob import has_magic +from pathlib import Path + +# for backwards compat, we export cache things from here too +from fsspec.caching import ( # noqa: F401 + BaseCache, + BlockCache, + BytesCache, + MMapCache, + ReadAheadCache, + caches, +) +from fsspec.compression import compr +from fsspec.config import conf +from fsspec.registry import filesystem, get_filesystem_class +from fsspec.utils import ( + _unstrip_protocol, + build_name_function, + infer_compression, + stringify_path, +) + +logger = logging.getLogger("fsspec") + + +class OpenFile: + """ + File-like object to be used in a context + + Can layer (buffered) text-mode and compression over any file-system, which + are typically binary-only. + + These instances are safe to serialize, as the low-level file object + is not created until invoked using ``with``. + + Parameters + ---------- + fs: FileSystem + The file system to use for opening the file. Should be a subclass or duck-type + with ``fsspec.spec.AbstractFileSystem`` + path: str + Location to open + mode: str like 'rb', optional + Mode of the opened file + compression: str or None, optional + Compression to apply + encoding: str or None, optional + The encoding to use if opened in text mode. + errors: str or None, optional + How to handle encoding errors if opened in text mode. + newline: None or str + Passed to TextIOWrapper in text mode, how to handle line endings. + autoopen: bool + If True, calls open() immediately. Mostly used by pickle + pos: int + If given and autoopen is True, seek to this location immediately + """ + + def __init__( + self, + fs, + path, + mode="rb", + compression=None, + encoding=None, + errors=None, + newline=None, + ): + self.fs = fs + self.path = path + self.mode = mode + self.compression = get_compression(path, compression) + self.encoding = encoding + self.errors = errors + self.newline = newline + self.fobjects = [] + + def __reduce__(self): + return ( + OpenFile, + ( + self.fs, + self.path, + self.mode, + self.compression, + self.encoding, + self.errors, + self.newline, + ), + ) + + def __repr__(self): + return f"" + + def __enter__(self): + mode = self.mode.replace("t", "").replace("b", "") + "b" + + try: + f = self.fs.open(self.path, mode=mode) + except FileNotFoundError as e: + if has_magic(self.path): + raise FileNotFoundError( + "%s not found. The URL contains glob characters: you maybe needed\n" + "to pass expand=True in fsspec.open() or the storage_options of \n" + "your library. You can also set the config value 'open_expand'\n" + "before import, or fsspec.core.DEFAULT_EXPAND at runtime, to True.", + self.path, + ) from e + raise + + self.fobjects = [f] + + if self.compression is not None: + compress = compr[self.compression] + f = compress(f, mode=mode[0]) + self.fobjects.append(f) + + if "b" not in self.mode: + # assume, for example, that 'r' is equivalent to 'rt' as in builtin + f = PickleableTextIOWrapper( + f, encoding=self.encoding, errors=self.errors, newline=self.newline + ) + self.fobjects.append(f) + + return self.fobjects[-1] + + def __exit__(self, *args): + self.close() + + @property + def full_name(self): + return _unstrip_protocol(self.path, self.fs) + + def open(self): + """Materialise this as a real open file without context + + The OpenFile object should be explicitly closed to avoid enclosed file + instances persisting. You must, therefore, keep a reference to the OpenFile + during the life of the file-like it generates. + """ + return self.__enter__() + + def close(self): + """Close all encapsulated file objects""" + for f in reversed(self.fobjects): + if "r" not in self.mode and not f.closed: + f.flush() + f.close() + self.fobjects.clear() + + +class OpenFiles(list): + """List of OpenFile instances + + Can be used in a single context, which opens and closes all of the + contained files. Normal list access to get the elements works as + normal. + + A special case is made for caching filesystems - the files will + be down/uploaded together at the start or end of the context, and + this may happen concurrently, if the target filesystem supports it. + """ + + def __init__(self, *args, mode="rb", fs=None): + self.mode = mode + self.fs = fs + self.files = [] + super().__init__(*args) + + def __enter__(self): + if self.fs is None: + raise ValueError("Context has already been used") + + fs = self.fs + while True: + if hasattr(fs, "open_many"): + # check for concurrent cache download; or set up for upload + self.files = fs.open_many(self) + return self.files + if hasattr(fs, "fs") and fs.fs is not None: + fs = fs.fs + else: + break + return [s.__enter__() for s in self] + + def __exit__(self, *args): + fs = self.fs + [s.__exit__(*args) for s in self] + if "r" not in self.mode: + while True: + if hasattr(fs, "open_many"): + # check for concurrent cache upload + fs.commit_many(self.files) + return + if hasattr(fs, "fs") and fs.fs is not None: + fs = fs.fs + else: + break + + def __getitem__(self, item): + out = super().__getitem__(item) + if isinstance(item, slice): + return OpenFiles(out, mode=self.mode, fs=self.fs) + return out + + def __repr__(self): + return f"" + + +def open_files( + urlpath, + mode="rb", + compression=None, + encoding="utf8", + errors=None, + name_function=None, + num=1, + protocol=None, + newline=None, + auto_mkdir=True, + expand=True, + **kwargs, +): + """Given a path or paths, return a list of ``OpenFile`` objects. + + For writing, a str path must contain the "*" character, which will be filled + in by increasing numbers, e.g., "part*" -> "part1", "part2" if num=2. + + For either reading or writing, can instead provide explicit list of paths. + + Parameters + ---------- + urlpath: string or list + Absolute or relative filepath(s). Prefix with a protocol like ``s3://`` + to read from alternative filesystems. To read from multiple files you + can pass a globstring or a list of paths, with the caveat that they + must all have the same protocol. + mode: 'rb', 'wt', etc. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding: str + For text mode only + errors: None or str + Passed to TextIOWrapper in text mode + name_function: function or None + if opening a set of files for writing, those files do not yet exist, + so we need to generate their names by formatting the urlpath for + each sequence number + num: int [1] + if writing mode, number of files we expect to create (passed to + name+function) + protocol: str or None + If given, overrides the protocol found in the URL. + newline: bytes or None + Used for line terminator in text mode. If None, uses system default; + if blank, uses no translation. + auto_mkdir: bool (True) + If in write mode, this will ensure the target directory exists before + writing, by calling ``fs.mkdirs(exist_ok=True)``. + expand: bool + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Examples + -------- + >>> files = open_files('2015-*-*.csv') # doctest: +SKIP + >>> files = open_files( + ... 's3://bucket/2015-*-*.csv.gz', compression='gzip' + ... ) # doctest: +SKIP + + Returns + ------- + An ``OpenFiles`` instance, which is a list of ``OpenFile`` objects that can + be used as a single context + + Notes + ----- + For a full list of the available protocols and the implementations that + they map across to see the latest online documentation: + + - For implementations built into ``fsspec`` see + https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations + - For implementations in separate packages see + https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations + """ + fs, fs_token, paths = get_fs_token_paths( + urlpath, + mode, + num=num, + name_function=name_function, + storage_options=kwargs, + protocol=protocol, + expand=expand, + ) + if fs.protocol == "file": + fs.auto_mkdir = auto_mkdir + elif "r" not in mode and auto_mkdir: + parents = {fs._parent(path) for path in paths} + for parent in parents: + try: + fs.makedirs(parent, exist_ok=True) + except PermissionError: + pass + return OpenFiles( + [ + OpenFile( + fs, + path, + mode=mode, + compression=compression, + encoding=encoding, + errors=errors, + newline=newline, + ) + for path in paths + ], + mode=mode, + fs=fs, + ) + + +def _un_chain(path, kwargs): + # Avoid a circular import + from fsspec.implementations.cached import CachingFileSystem + + if "::" in path: + x = re.compile(".*[^a-z]+.*") # test for non protocol-like single word + bits = [] + for p in path.split("::"): + if "://" in p or x.match(p): + bits.append(p) + else: + bits.append(p + "://") + else: + bits = [path] + # [[url, protocol, kwargs], ...] + out = [] + previous_bit = None + kwargs = kwargs.copy() + for bit in reversed(bits): + protocol = kwargs.pop("protocol", None) or split_protocol(bit)[0] or "file" + cls = get_filesystem_class(protocol) + extra_kwargs = cls._get_kwargs_from_urls(bit) + kws = kwargs.pop(protocol, {}) + if bit is bits[0]: + kws.update(kwargs) + kw = dict( + **{k: v for k, v in extra_kwargs.items() if k not in kws or v != kws[k]}, + **kws, + ) + bit = cls._strip_protocol(bit) + if "target_protocol" not in kw and issubclass(cls, CachingFileSystem): + bit = previous_bit + out.append((bit, protocol, kw)) + previous_bit = bit + out.reverse() + return out + + +def url_to_fs(url, **kwargs): + """ + Turn fully-qualified and potentially chained URL into filesystem instance + + Parameters + ---------- + url : str + The fsspec-compatible URL + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Returns + ------- + filesystem : FileSystem + The new filesystem discovered from ``url`` and created with + ``**kwargs``. + urlpath : str + The file-systems-specific URL for ``url``. + """ + url = stringify_path(url) + # non-FS arguments that appear in fsspec.open() + # inspect could keep this in sync with open()'s signature + known_kwargs = { + "compression", + "encoding", + "errors", + "expand", + "mode", + "name_function", + "newline", + "num", + } + kwargs = {k: v for k, v in kwargs.items() if k not in known_kwargs} + chain = _un_chain(url, kwargs) + inkwargs = {} + # Reverse iterate the chain, creating a nested target_* structure + for i, ch in enumerate(reversed(chain)): + urls, protocol, kw = ch + if i == len(chain) - 1: + inkwargs = dict(**kw, **inkwargs) + continue + inkwargs["target_options"] = dict(**kw, **inkwargs) + inkwargs["target_protocol"] = protocol + inkwargs["fo"] = urls + urlpath, protocol, _ = chain[0] + fs = filesystem(protocol, **inkwargs) + return fs, urlpath + + +DEFAULT_EXPAND = conf.get("open_expand", False) + + +def open( + urlpath, + mode="rb", + compression=None, + encoding="utf8", + errors=None, + protocol=None, + newline=None, + expand=None, + **kwargs, +): + """Given a path or paths, return one ``OpenFile`` object. + + Parameters + ---------- + urlpath: string or list + Absolute or relative filepath. Prefix with a protocol like ``s3://`` + to read from alternative filesystems. Should not include glob + character(s). + mode: 'rb', 'wt', etc. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding: str + For text mode only + errors: None or str + Passed to TextIOWrapper in text mode + protocol: str or None + If given, overrides the protocol found in the URL. + newline: bytes or None + Used for line terminator in text mode. If None, uses system default; + if blank, uses no translation. + expand: bool or None + Whether to regard file paths containing special glob characters as needing + expansion (finding the first match) or absolute. Setting False allows using + paths which do embed such characters. If None (default), this argument + takes its value from the DEFAULT_EXPAND module variable, which takes + its initial value from the "open_expand" config value at startup, which will + be False if not set. + **kwargs: dict + Extra options that make sense to a particular storage connection, e.g. + host, port, username, password, etc. + + Examples + -------- + >>> openfile = open('2015-01-01.csv') # doctest: +SKIP + >>> openfile = open( + ... 's3://bucket/2015-01-01.csv.gz', compression='gzip' + ... ) # doctest: +SKIP + >>> with openfile as f: + ... df = pd.read_csv(f) # doctest: +SKIP + ... + + Returns + ------- + ``OpenFile`` object. + + Notes + ----- + For a full list of the available protocols and the implementations that + they map across to see the latest online documentation: + + - For implementations built into ``fsspec`` see + https://filesystem-spec.readthedocs.io/en/latest/api.html#built-in-implementations + - For implementations in separate packages see + https://filesystem-spec.readthedocs.io/en/latest/api.html#other-known-implementations + """ + expand = DEFAULT_EXPAND if expand is None else expand + out = open_files( + urlpath=[urlpath], + mode=mode, + compression=compression, + encoding=encoding, + errors=errors, + protocol=protocol, + newline=newline, + expand=expand, + **kwargs, + ) + if not out: + raise FileNotFoundError(urlpath) + return out[0] + + +def open_local( + url: str | list[str] | Path | list[Path], + mode: str = "rb", + **storage_options: dict, +) -> str | list[str]: + """Open file(s) which can be resolved to local + + For files which either are local, or get downloaded upon open + (e.g., by file caching) + + Parameters + ---------- + url: str or list(str) + mode: str + Must be read mode + storage_options: + passed on to FS for or used by open_files (e.g., compression) + """ + if "r" not in mode: + raise ValueError("Can only ensure local files when reading") + of = open_files(url, mode=mode, **storage_options) + if not getattr(of[0].fs, "local_file", False): + raise ValueError( + "open_local can only be used on a filesystem which" + " has attribute local_file=True" + ) + with of as files: + paths = [f.name for f in files] + if (isinstance(url, str) and not has_magic(url)) or isinstance(url, Path): + return paths[0] + return paths + + +def get_compression(urlpath, compression): + if compression == "infer": + compression = infer_compression(urlpath) + if compression is not None and compression not in compr: + raise ValueError(f"Compression type {compression} not supported") + return compression + + +def split_protocol(urlpath): + """Return protocol, path pair""" + urlpath = stringify_path(urlpath) + if "://" in urlpath: + protocol, path = urlpath.split("://", 1) + if len(protocol) > 1: + # excludes Windows paths + return protocol, path + if urlpath.startswith("data:"): + return urlpath.split(":", 1) + return None, urlpath + + +def strip_protocol(urlpath): + """Return only path part of full URL, according to appropriate backend""" + protocol, _ = split_protocol(urlpath) + cls = get_filesystem_class(protocol) + return cls._strip_protocol(urlpath) + + +def expand_paths_if_needed(paths, mode, num, fs, name_function): + """Expand paths if they have a ``*`` in them (write mode) or any of ``*?[]`` + in them (read mode). + + :param paths: list of paths + mode: str + Mode in which to open files. + num: int + If opening in writing mode, number of files we expect to create. + fs: filesystem object + name_function: callable + If opening in writing mode, this callable is used to generate path + names. Names are generated for each partition by + ``urlpath.replace('*', name_function(partition_index))``. + :return: list of paths + """ + expanded_paths = [] + paths = list(paths) + + if "w" in mode: # read mode + if sum(1 for p in paths if "*" in p) > 1: + raise ValueError( + "When writing data, only one filename mask can be specified." + ) + num = max(num, len(paths)) + + for curr_path in paths: + if "*" in curr_path: + # expand using name_function + expanded_paths.extend(_expand_paths(curr_path, name_function, num)) + else: + expanded_paths.append(curr_path) + # if we generated more paths that asked for, trim the list + if len(expanded_paths) > num: + expanded_paths = expanded_paths[:num] + + else: # read mode + for curr_path in paths: + if has_magic(curr_path): + # expand using glob + expanded_paths.extend(fs.glob(curr_path)) + else: + expanded_paths.append(curr_path) + + return expanded_paths + + +def get_fs_token_paths( + urlpath, + mode="rb", + num=1, + name_function=None, + storage_options=None, + protocol=None, + expand=True, +): + """Filesystem, deterministic token, and paths from a urlpath and options. + + Parameters + ---------- + urlpath: string or iterable + Absolute or relative filepath, URL (may include protocols like + ``s3://``), or globstring pointing to data. + mode: str, optional + Mode in which to open files. + num: int, optional + If opening in writing mode, number of files we expect to create. + name_function: callable, optional + If opening in writing mode, this callable is used to generate path + names. Names are generated for each partition by + ``urlpath.replace('*', name_function(partition_index))``. + storage_options: dict, optional + Additional keywords to pass to the filesystem class. + protocol: str or None + To override the protocol specifier in the URL + expand: bool + Expand string paths for writing, assuming the path is a directory + """ + if isinstance(urlpath, (list, tuple, set)): + if not urlpath: + raise ValueError("empty urlpath sequence") + urlpath0 = stringify_path(next(iter(urlpath))) + else: + urlpath0 = stringify_path(urlpath) + storage_options = storage_options or {} + if protocol: + storage_options["protocol"] = protocol + chain = _un_chain(urlpath0, storage_options or {}) + inkwargs = {} + # Reverse iterate the chain, creating a nested target_* structure + for i, ch in enumerate(reversed(chain)): + urls, nested_protocol, kw = ch + if i == len(chain) - 1: + inkwargs = dict(**kw, **inkwargs) + continue + inkwargs["target_options"] = dict(**kw, **inkwargs) + inkwargs["target_protocol"] = nested_protocol + inkwargs["fo"] = urls + paths, protocol, _ = chain[0] + fs = filesystem(protocol, **inkwargs) + if isinstance(urlpath, (list, tuple, set)): + pchains = [ + _un_chain(stringify_path(u), storage_options or {})[0] for u in urlpath + ] + if len({pc[1] for pc in pchains}) > 1: + raise ValueError("Protocol mismatch getting fs from %s", urlpath) + paths = [pc[0] for pc in pchains] + else: + paths = fs._strip_protocol(paths) + if isinstance(paths, (list, tuple, set)): + if expand: + paths = expand_paths_if_needed(paths, mode, num, fs, name_function) + elif not isinstance(paths, list): + paths = list(paths) + else: + if ("w" in mode or "x" in mode) and expand: + paths = _expand_paths(paths, name_function, num) + elif "*" in paths: + paths = [f for f in sorted(fs.glob(paths)) if not fs.isdir(f)] + else: + paths = [paths] + + return fs, fs._fs_token, paths + + +def _expand_paths(path, name_function, num): + if isinstance(path, str): + if path.count("*") > 1: + raise ValueError("Output path spec must contain exactly one '*'.") + elif "*" not in path: + path = os.path.join(path, "*.part") + + if name_function is None: + name_function = build_name_function(num - 1) + + paths = [path.replace("*", name_function(i)) for i in range(num)] + if paths != sorted(paths): + logger.warning( + "In order to preserve order between partitions" + " paths created with ``name_function`` should " + "sort to partition order" + ) + elif isinstance(path, (tuple, list)): + assert len(path) == num + paths = list(path) + else: + raise ValueError( + "Path should be either\n" + "1. A list of paths: ['foo.json', 'bar.json', ...]\n" + "2. A directory: 'foo/\n" + "3. A path with a '*' in it: 'foo.*.json'" + ) + return paths + + +class PickleableTextIOWrapper(io.TextIOWrapper): + """TextIOWrapper cannot be pickled. This solves it. + + Requires that ``buffer`` be pickleable, which all instances of + AbstractBufferedFile are. + """ + + def __init__( + self, + buffer, + encoding=None, + errors=None, + newline=None, + line_buffering=False, + write_through=False, + ): + self.args = buffer, encoding, errors, newline, line_buffering, write_through + super().__init__(*self.args) + + def __reduce__(self): + return PickleableTextIOWrapper, self.args diff --git a/lib/python3.12/site-packages/fsspec/dircache.py b/lib/python3.12/site-packages/fsspec/dircache.py new file mode 100644 index 0000000000000000000000000000000000000000..eca19566b135e5a7a4f6e7407d56411ec58bfe44 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/dircache.py @@ -0,0 +1,98 @@ +import time +from collections.abc import MutableMapping +from functools import lru_cache + + +class DirCache(MutableMapping): + """ + Caching of directory listings, in a structure like:: + + {"path0": [ + {"name": "path0/file0", + "size": 123, + "type": "file", + ... + }, + {"name": "path0/file1", + }, + ... + ], + "path1": [...] + } + + Parameters to this class control listing expiry or indeed turn + caching off + """ + + def __init__( + self, + use_listings_cache=True, + listings_expiry_time=None, + max_paths=None, + **kwargs, + ): + """ + + Parameters + ---------- + use_listings_cache: bool + If False, this cache never returns items, but always reports KeyError, + and setting items has no effect + listings_expiry_time: int or float (optional) + Time in seconds that a listing is considered valid. If None, + listings do not expire. + max_paths: int (optional) + The number of most recent listings that are considered valid; 'recent' + refers to when the entry was set. + """ + self._cache = {} + self._times = {} + if max_paths: + self._q = lru_cache(max_paths + 1)(lambda key: self._cache.pop(key, None)) + self.use_listings_cache = use_listings_cache + self.listings_expiry_time = listings_expiry_time + self.max_paths = max_paths + + def __getitem__(self, item): + if self.listings_expiry_time is not None: + if self._times.get(item, 0) - time.time() < -self.listings_expiry_time: + del self._cache[item] + if self.max_paths: + self._q(item) + return self._cache[item] # maybe raises KeyError + + def clear(self): + self._cache.clear() + + def __len__(self): + return len(self._cache) + + def __contains__(self, item): + try: + self[item] + return True + except KeyError: + return False + + def __setitem__(self, key, value): + if not self.use_listings_cache: + return + if self.max_paths: + self._q(key) + self._cache[key] = value + if self.listings_expiry_time is not None: + self._times[key] = time.time() + + def __delitem__(self, key): + del self._cache[key] + + def __iter__(self): + entries = list(self._cache) + + return (k for k in entries if k in self) + + def __reduce__(self): + return ( + DirCache, + (self.use_listings_cache, self.listings_expiry_time, self.max_paths), + ) diff --git a/lib/python3.12/site-packages/fsspec/exceptions.py b/lib/python3.12/site-packages/fsspec/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..ae8905475f02655f4fc5863931d99ca9da55db78 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/exceptions.py @@ -0,0 +1,18 @@ +""" +fsspec user-defined exception classes +""" + +import asyncio + + +class BlocksizeMismatchError(ValueError): + """ + Raised when a cached file is opened with a different blocksize than it was + written with + """ + + +class FSTimeoutError(asyncio.TimeoutError): + """ + Raised when a fsspec function timed out occurs + """ diff --git a/lib/python3.12/site-packages/fsspec/fuse.py b/lib/python3.12/site-packages/fsspec/fuse.py new file mode 100644 index 0000000000000000000000000000000000000000..566d520fce3e94e3bbaee48c3c6acc9f1db315a8 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/fuse.py @@ -0,0 +1,324 @@ +import argparse +import logging +import os +import stat +import threading +import time +from errno import EIO, ENOENT + +from fuse import FUSE, FuseOSError, LoggingMixIn, Operations + +from fsspec import __version__ +from fsspec.core import url_to_fs + +logger = logging.getLogger("fsspec.fuse") + + +class FUSEr(Operations): + def __init__(self, fs, path, ready_file=False): + self.fs = fs + self.cache = {} + self.root = path.rstrip("/") + "/" + self.counter = 0 + logger.info("Starting FUSE at %s", path) + self._ready_file = ready_file + + def getattr(self, path, fh=None): + logger.debug("getattr %s", path) + if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: + return {"type": "file", "st_size": 5} + + path = "".join([self.root, path.lstrip("/")]).rstrip("/") + try: + info = self.fs.info(path) + except FileNotFoundError as exc: + raise FuseOSError(ENOENT) from exc + + data = {"st_uid": info.get("uid", 1000), "st_gid": info.get("gid", 1000)} + perm = info.get("mode", 0o777) + + if info["type"] != "file": + data["st_mode"] = stat.S_IFDIR | perm + data["st_size"] = 0 + data["st_blksize"] = 0 + else: + data["st_mode"] = stat.S_IFREG | perm + data["st_size"] = info["size"] + data["st_blksize"] = 5 * 2**20 + data["st_nlink"] = 1 + data["st_atime"] = info["atime"] if "atime" in info else time.time() + data["st_ctime"] = info["ctime"] if "ctime" in info else time.time() + data["st_mtime"] = info["mtime"] if "mtime" in info else time.time() + return data + + def readdir(self, path, fh): + logger.debug("readdir %s", path) + path = "".join([self.root, path.lstrip("/")]) + files = self.fs.ls(path, False) + files = [os.path.basename(f.rstrip("/")) for f in files] + return [".", ".."] + files + + def mkdir(self, path, mode): + path = "".join([self.root, path.lstrip("/")]) + self.fs.mkdir(path) + return 0 + + def rmdir(self, path): + path = "".join([self.root, path.lstrip("/")]) + self.fs.rmdir(path) + return 0 + + def read(self, path, size, offset, fh): + logger.debug("read %s", (path, size, offset)) + if self._ready_file and path in ["/.fuse_ready", ".fuse_ready"]: + # status indicator + return b"ready" + + f = self.cache[fh] + f.seek(offset) + out = f.read(size) + return out + + def write(self, path, data, offset, fh): + logger.debug("write %s", (path, offset)) + f = self.cache[fh] + f.seek(offset) + f.write(data) + return len(data) + + def create(self, path, flags, fi=None): + logger.debug("create %s", (path, flags)) + fn = "".join([self.root, path.lstrip("/")]) + self.fs.touch(fn) # OS will want to get attributes immediately + f = self.fs.open(fn, "wb") + self.cache[self.counter] = f + self.counter += 1 + return self.counter - 1 + + def open(self, path, flags): + logger.debug("open %s", (path, flags)) + fn = "".join([self.root, path.lstrip("/")]) + if flags % 2 == 0: + # read + mode = "rb" + else: + # write/create + mode = "wb" + self.cache[self.counter] = self.fs.open(fn, mode) + self.counter += 1 + return self.counter - 1 + + def truncate(self, path, length, fh=None): + fn = "".join([self.root, path.lstrip("/")]) + if length != 0: + raise NotImplementedError + # maybe should be no-op since open with write sets size to zero anyway + self.fs.touch(fn) + + def unlink(self, path): + fn = "".join([self.root, path.lstrip("/")]) + try: + self.fs.rm(fn, False) + except (OSError, FileNotFoundError) as exc: + raise FuseOSError(EIO) from exc + + def release(self, path, fh): + try: + if fh in self.cache: + f = self.cache[fh] + f.close() + self.cache.pop(fh) + except Exception as e: + print(e) + return 0 + + def chmod(self, path, mode): + if hasattr(self.fs, "chmod"): + path = "".join([self.root, path.lstrip("/")]) + return self.fs.chmod(path, mode) + raise NotImplementedError + + +def run( + fs, + path, + mount_point, + foreground=True, + threads=False, + ready_file=False, + ops_class=FUSEr, +): + """Mount stuff in a local directory + + This uses fusepy to make it appear as if a given path on an fsspec + instance is in fact resident within the local file-system. + + This requires that fusepy by installed, and that FUSE be available on + the system (typically requiring a package to be installed with + apt, yum, brew, etc.). + + Parameters + ---------- + fs: file-system instance + From one of the compatible implementations + path: str + Location on that file-system to regard as the root directory to + mount. Note that you typically should include the terminating "/" + character. + mount_point: str + An empty directory on the local file-system where the contents of + the remote path will appear. + foreground: bool + Whether or not calling this function will block. Operation will + typically be more stable if True. + threads: bool + Whether or not to create threads when responding to file operations + within the mounter directory. Operation will typically be more + stable if False. + ready_file: bool + Whether the FUSE process is ready. The ``.fuse_ready`` file will + exist in the ``mount_point`` directory if True. Debugging purpose. + ops_class: FUSEr or Subclass of FUSEr + To override the default behavior of FUSEr. For Example, logging + to file. + + """ + func = lambda: FUSE( + ops_class(fs, path, ready_file=ready_file), + mount_point, + nothreads=not threads, + foreground=foreground, + ) + if not foreground: + th = threading.Thread(target=func) + th.daemon = True + th.start() + return th + else: # pragma: no cover + try: + func() + except KeyboardInterrupt: + pass + + +def main(args): + """Mount filesystem from chained URL to MOUNT_POINT. + + Examples: + + python3 -m fsspec.fuse memory /usr/share /tmp/mem + + python3 -m fsspec.fuse local /tmp/source /tmp/local \\ + -l /tmp/fsspecfuse.log + + You can also mount chained-URLs and use special settings: + + python3 -m fsspec.fuse 'filecache::zip::file://data.zip' \\ + / /tmp/zip \\ + -o 'filecache-cache_storage=/tmp/simplecache' + + You can specify the type of the setting by using `[int]` or `[bool]`, + (`true`, `yes`, `1` represents the Boolean value `True`): + + python3 -m fsspec.fuse 'simplecache::ftp://ftp1.at.proftpd.org' \\ + /historic/packages/RPMS /tmp/ftp \\ + -o 'simplecache-cache_storage=/tmp/simplecache' \\ + -o 'simplecache-check_files=false[bool]' \\ + -o 'ftp-listings_expiry_time=60[int]' \\ + -o 'ftp-username=anonymous' \\ + -o 'ftp-password=xieyanbo' + """ + + class RawDescriptionArgumentParser(argparse.ArgumentParser): + def format_help(self): + usage = super().format_help() + parts = usage.split("\n\n") + parts[1] = self.description.rstrip() + return "\n\n".join(parts) + + parser = RawDescriptionArgumentParser(prog="fsspec.fuse", description=main.__doc__) + parser.add_argument("--version", action="version", version=__version__) + parser.add_argument("url", type=str, help="fs url") + parser.add_argument("source_path", type=str, help="source directory in fs") + parser.add_argument("mount_point", type=str, help="local directory") + parser.add_argument( + "-o", + "--option", + action="append", + help="Any options of protocol included in the chained URL", + ) + parser.add_argument( + "-l", "--log-file", type=str, help="Logging FUSE debug info (Default: '')" + ) + parser.add_argument( + "-f", + "--foreground", + action="store_false", + help="Running in foreground or not (Default: False)", + ) + parser.add_argument( + "-t", + "--threads", + action="store_false", + help="Running with threads support (Default: False)", + ) + parser.add_argument( + "-r", + "--ready-file", + action="store_false", + help="The `.fuse_ready` file will exist after FUSE is ready. " + "(Debugging purpose, Default: False)", + ) + args = parser.parse_args(args) + + kwargs = {} + for item in args.option or []: + key, sep, value = item.partition("=") + if not sep: + parser.error(message=f"Wrong option: {item!r}") + val = value.lower() + if val.endswith("[int]"): + value = int(value[: -len("[int]")]) + elif val.endswith("[bool]"): + value = val[: -len("[bool]")] in ["1", "yes", "true"] + + if "-" in key: + fs_name, setting_name = key.split("-", 1) + if fs_name in kwargs: + kwargs[fs_name][setting_name] = value + else: + kwargs[fs_name] = {setting_name: value} + else: + kwargs[key] = value + + if args.log_file: + logging.basicConfig( + level=logging.DEBUG, + filename=args.log_file, + format="%(asctime)s %(message)s", + ) + + class LoggingFUSEr(FUSEr, LoggingMixIn): + pass + + fuser = LoggingFUSEr + else: + fuser = FUSEr + + fs, url_path = url_to_fs(args.url, **kwargs) + logger.debug("Mounting %s to %s", url_path, str(args.mount_point)) + run( + fs, + args.source_path, + args.mount_point, + foreground=args.foreground, + threads=args.threads, + ready_file=args.ready_file, + ops_class=fuser, + ) + + +if __name__ == "__main__": + import sys + + main(sys.argv[1:]) diff --git a/lib/python3.12/site-packages/fsspec/generic.py b/lib/python3.12/site-packages/fsspec/generic.py new file mode 100644 index 0000000000000000000000000000000000000000..9bad0f048f737400b4cd919f7699d15fbfbf7c62 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/generic.py @@ -0,0 +1,411 @@ +from __future__ import annotations + +import inspect +import logging +import os +import shutil +import uuid +from typing import Optional + +from .asyn import AsyncFileSystem, _run_coros_in_chunks, sync_wrapper +from .callbacks import DEFAULT_CALLBACK +from .core import filesystem, get_filesystem_class, split_protocol, url_to_fs + +_generic_fs = {} +logger = logging.getLogger("fsspec.generic") + + +def set_generic_fs(protocol, **storage_options): + _generic_fs[protocol] = filesystem(protocol, **storage_options) + + +default_method = "default" + + +def _resolve_fs(url, method=None, protocol=None, storage_options=None): + """Pick instance of backend FS""" + method = method or default_method + protocol = protocol or split_protocol(url)[0] + storage_options = storage_options or {} + if method == "default": + return filesystem(protocol) + if method == "generic": + return _generic_fs[protocol] + if method == "current": + cls = get_filesystem_class(protocol) + return cls.current() + if method == "options": + fs, _ = url_to_fs(url, **storage_options.get(protocol, {})) + return fs + raise ValueError(f"Unknown FS resolution method: {method}") + + +def rsync( + source, + destination, + delete_missing=False, + source_field="size", + dest_field="size", + update_cond="different", + inst_kwargs=None, + fs=None, + **kwargs, +): + """Sync files between two directory trees + + (experimental) + + Parameters + ---------- + source: str + Root of the directory tree to take files from. This must be a directory, but + do not include any terminating "/" character + destination: str + Root path to copy into. The contents of this location should be + identical to the contents of ``source`` when done. This will be made a + directory, and the terminal "/" should not be included. + delete_missing: bool + If there are paths in the destination that don't exist in the + source and this is True, delete them. Otherwise, leave them alone. + source_field: str | callable + If ``update_field`` is "different", this is the key in the info + of source files to consider for difference. Maybe a function of the + info dict. + dest_field: str | callable + If ``update_field`` is "different", this is the key in the info + of destination files to consider for difference. May be a function of + the info dict. + update_cond: "different"|"always"|"never" + If "always", every file is copied, regardless of whether it exists in + the destination. If "never", files that exist in the destination are + not copied again. If "different" (default), only copy if the info + fields given by ``source_field`` and ``dest_field`` (usually "size") + are different. Other comparisons may be added in the future. + inst_kwargs: dict|None + If ``fs`` is None, use this set of keyword arguments to make a + GenericFileSystem instance + fs: GenericFileSystem|None + Instance to use if explicitly given. The instance defines how to + to make downstream file system instances from paths. + + Returns + ------- + dict of the copy operations that were performed, {source: destination} + """ + fs = fs or GenericFileSystem(**(inst_kwargs or {})) + source = fs._strip_protocol(source) + destination = fs._strip_protocol(destination) + allfiles = fs.find(source, withdirs=True, detail=True) + if not fs.isdir(source): + raise ValueError("Can only rsync on a directory") + otherfiles = fs.find(destination, withdirs=True, detail=True) + dirs = [ + a + for a, v in allfiles.items() + if v["type"] == "directory" and a.replace(source, destination) not in otherfiles + ] + logger.debug(f"{len(dirs)} directories to create") + if dirs: + fs.make_many_dirs( + [dirn.replace(source, destination) for dirn in dirs], exist_ok=True + ) + allfiles = {a: v for a, v in allfiles.items() if v["type"] == "file"} + logger.debug(f"{len(allfiles)} files to consider for copy") + to_delete = [ + o + for o, v in otherfiles.items() + if o.replace(destination, source) not in allfiles and v["type"] == "file" + ] + for k, v in allfiles.copy().items(): + otherfile = k.replace(source, destination) + if otherfile in otherfiles: + if update_cond == "always": + allfiles[k] = otherfile + elif update_cond == "different": + inf1 = source_field(v) if callable(source_field) else v[source_field] + v2 = otherfiles[otherfile] + inf2 = dest_field(v2) if callable(dest_field) else v2[dest_field] + if inf1 != inf2: + # details mismatch, make copy + allfiles[k] = otherfile + else: + # details match, don't copy + allfiles.pop(k) + else: + # file not in target yet + allfiles[k] = otherfile + logger.debug(f"{len(allfiles)} files to copy") + if allfiles: + source_files, target_files = zip(*allfiles.items()) + fs.cp(source_files, target_files, **kwargs) + logger.debug(f"{len(to_delete)} files to delete") + if delete_missing and to_delete: + fs.rm(to_delete) + return allfiles + + +class GenericFileSystem(AsyncFileSystem): + """Wrapper over all other FS types + + + + This implementation is a single unified interface to be able to run FS operations + over generic URLs, and dispatch to the specific implementations using the URL + protocol prefix. + + Note: instances of this FS are always async, even if you never use it with any async + backend. + """ + + protocol = "generic" # there is no real reason to ever use a protocol with this FS + + def __init__(self, default_method="default", **kwargs): + """ + + Parameters + ---------- + default_method: str (optional) + Defines how to configure backend FS instances. Options are: + - "default": instantiate like FSClass(), with no + extra arguments; this is the default instance of that FS, and can be + configured via the config system + - "generic": takes instances from the `_generic_fs` dict in this module, + which you must populate before use. Keys are by protocol + - "current": takes the most recently instantiated version of each FS + """ + self.method = default_method + super().__init__(**kwargs) + + def _parent(self, path): + fs = _resolve_fs(path, self.method) + return fs.unstrip_protocol(fs._parent(path)) + + def _strip_protocol(self, path): + # normalization only + fs = _resolve_fs(path, self.method) + return fs.unstrip_protocol(fs._strip_protocol(path)) + + async def _find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + fs = _resolve_fs(path, self.method) + if fs.async_impl: + out = await fs._find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs + ) + else: + out = fs.find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=True, **kwargs + ) + result = {} + for k, v in out.items(): + v = v.copy() # don't corrupt target FS dircache + name = fs.unstrip_protocol(k) + v["name"] = name + result[name] = v + if detail: + return result + return list(result) + + async def _info(self, url, **kwargs): + fs = _resolve_fs(url, self.method) + if fs.async_impl: + out = await fs._info(url, **kwargs) + else: + out = fs.info(url, **kwargs) + out = out.copy() # don't edit originals + out["name"] = fs.unstrip_protocol(out["name"]) + return out + + async def _ls( + self, + url, + detail=True, + **kwargs, + ): + fs = _resolve_fs(url, self.method) + if fs.async_impl: + out = await fs._ls(url, detail=True, **kwargs) + else: + out = fs.ls(url, detail=True, **kwargs) + out = [o.copy() for o in out] # don't edit originals + for o in out: + o["name"] = fs.unstrip_protocol(o["name"]) + if detail: + return out + else: + return [o["name"] for o in out] + + async def _cat_file( + self, + url, + **kwargs, + ): + fs = _resolve_fs(url, self.method) + if fs.async_impl: + return await fs._cat_file(url, **kwargs) + else: + return fs.cat_file(url, **kwargs) + + async def _pipe_file( + self, + path, + value, + **kwargs, + ): + fs = _resolve_fs(path, self.method) + if fs.async_impl: + return await fs._pipe_file(path, value, **kwargs) + else: + return fs.pipe_file(path, value, **kwargs) + + async def _rm(self, url, **kwargs): + urls = url + if isinstance(urls, str): + urls = [urls] + fs = _resolve_fs(urls[0], self.method) + if fs.async_impl: + await fs._rm(urls, **kwargs) + else: + fs.rm(url, **kwargs) + + async def _makedirs(self, path, exist_ok=False): + logger.debug("Make dir %s", path) + fs = _resolve_fs(path, self.method) + if fs.async_impl: + await fs._makedirs(path, exist_ok=exist_ok) + else: + fs.makedirs(path, exist_ok=exist_ok) + + def rsync(self, source, destination, **kwargs): + """Sync files between two directory trees + + See `func:rsync` for more details. + """ + rsync(source, destination, fs=self, **kwargs) + + async def _cp_file( + self, + url, + url2, + blocksize=2**20, + callback=DEFAULT_CALLBACK, + **kwargs, + ): + fs = _resolve_fs(url, self.method) + fs2 = _resolve_fs(url2, self.method) + if fs is fs2: + # pure remote + if fs.async_impl: + return await fs._cp_file(url, url2, **kwargs) + else: + return fs.cp_file(url, url2, **kwargs) + kw = {"blocksize": 0, "cache_type": "none"} + try: + f1 = ( + await fs.open_async(url, "rb") + if hasattr(fs, "open_async") + else fs.open(url, "rb", **kw) + ) + callback.set_size(await maybe_await(f1.size)) + f2 = ( + await fs2.open_async(url2, "wb") + if hasattr(fs2, "open_async") + else fs2.open(url2, "wb", **kw) + ) + while f1.size is None or f2.tell() < f1.size: + data = await maybe_await(f1.read(blocksize)) + if f1.size is None and not data: + break + await maybe_await(f2.write(data)) + callback.absolute_update(f2.tell()) + finally: + try: + await maybe_await(f2.close()) + await maybe_await(f1.close()) + except NameError: + # fail while opening f1 or f2 + pass + + async def _make_many_dirs(self, urls, exist_ok=True): + fs = _resolve_fs(urls[0], self.method) + if fs.async_impl: + coros = [fs._makedirs(u, exist_ok=exist_ok) for u in urls] + await _run_coros_in_chunks(coros) + else: + for u in urls: + fs.makedirs(u, exist_ok=exist_ok) + + make_many_dirs = sync_wrapper(_make_many_dirs) + + async def _copy( + self, + path1: list[str], + path2: list[str], + recursive: bool = False, + on_error: str = "ignore", + maxdepth: Optional[int] = None, + batch_size: Optional[int] = None, + tempdir: Optional[str] = None, + **kwargs, + ): + if recursive: + raise NotImplementedError + fs = _resolve_fs(path1[0], self.method) + fs2 = _resolve_fs(path2[0], self.method) + # not expanding paths atm., assume call is from rsync() + if fs is fs2: + # pure remote + if fs.async_impl: + return await fs._copy(path1, path2, **kwargs) + else: + return fs.copy(path1, path2, **kwargs) + await copy_file_op( + fs, path1, fs2, path2, tempdir, batch_size, on_error=on_error + ) + + +async def copy_file_op( + fs1, url1, fs2, url2, tempdir=None, batch_size=20, on_error="ignore" +): + import tempfile + + tempdir = tempdir or tempfile.mkdtemp() + try: + coros = [ + _copy_file_op( + fs1, + u1, + fs2, + u2, + os.path.join(tempdir, uuid.uuid4().hex), + on_error=on_error, + ) + for u1, u2 in zip(url1, url2) + ] + await _run_coros_in_chunks(coros, batch_size=batch_size) + finally: + shutil.rmtree(tempdir) + + +async def _copy_file_op(fs1, url1, fs2, url2, local, on_error="ignore"): + ex = () if on_error == "raise" else Exception + logger.debug("Copy %s -> %s", url1, url2) + try: + if fs1.async_impl: + await fs1._get_file(url1, local) + else: + fs1.get_file(url1, local) + if fs2.async_impl: + await fs2._put_file(local, url2) + else: + fs2.put_file(local, url2) + os.unlink(local) + logger.debug("Copy %s -> %s; done", url1, url2) + except ex as e: + logger.debug("ignoring cp exception for %s: %s", url1, e) + + +async def maybe_await(cor): + if inspect.iscoroutine(cor): + return await cor + else: + return cor diff --git a/lib/python3.12/site-packages/fsspec/gui.py b/lib/python3.12/site-packages/fsspec/gui.py new file mode 100644 index 0000000000000000000000000000000000000000..321379eb8ef924302cc2f56bccb3bdb873ff86e3 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/gui.py @@ -0,0 +1,416 @@ +import ast +import contextlib +import logging +import os +import re +from typing import ClassVar, Sequence + +import panel as pn + +from .core import OpenFile, get_filesystem_class, split_protocol +from .registry import known_implementations + +pn.extension() +logger = logging.getLogger("fsspec.gui") + + +class SigSlot: + """Signal-slot mixin, for Panel event passing + + Include this class in a widget manager's superclasses to be able to + register events and callbacks on Panel widgets managed by that class. + + The method ``_register`` should be called as widgets are added, and external + code should call ``connect`` to associate callbacks. + + By default, all signals emit a DEBUG logging statement. + """ + + # names of signals that this class may emit each of which must be + # set by _register for any new instance + signals: ClassVar[Sequence[str]] = [] + # names of actions that this class may respond to + slots: ClassVar[Sequence[str]] = [] + + # each of which must be a method name + + def __init__(self): + self._ignoring_events = False + self._sigs = {} + self._map = {} + self._setup() + + def _setup(self): + """Create GUI elements and register signals""" + self.panel = pn.pane.PaneBase() + # no signals to set up in the base class + + def _register( + self, widget, name, thing="value", log_level=logging.DEBUG, auto=False + ): + """Watch the given attribute of a widget and assign it a named event + + This is normally called at the time a widget is instantiated, in the + class which owns it. + + Parameters + ---------- + widget : pn.layout.Panel or None + Widget to watch. If None, an anonymous signal not associated with + any widget. + name : str + Name of this event + thing : str + Attribute of the given widget to watch + log_level : int + When the signal is triggered, a logging event of the given level + will be fired in the dfviz logger. + auto : bool + If True, automatically connects with a method in this class of the + same name. + """ + if name not in self.signals: + raise ValueError(f"Attempt to assign an undeclared signal: {name}") + self._sigs[name] = { + "widget": widget, + "callbacks": [], + "thing": thing, + "log": log_level, + } + wn = "-".join( + [ + getattr(widget, "name", str(widget)) if widget is not None else "none", + thing, + ] + ) + self._map[wn] = name + if widget is not None: + widget.param.watch(self._signal, thing, onlychanged=True) + if auto and hasattr(self, name): + self.connect(name, getattr(self, name)) + + def _repr_mimebundle_(self, *args, **kwargs): + """Display in a notebook or a server""" + try: + return self.panel._repr_mimebundle_(*args, **kwargs) + except (ValueError, AttributeError) as exc: + raise NotImplementedError( + "Panel does not seem to be set up properly" + ) from exc + + def connect(self, signal, slot): + """Associate call back with given event + + The callback must be a function which takes the "new" value of the + watched attribute as the only parameter. If the callback return False, + this cancels any further processing of the given event. + + Alternatively, the callback can be a string, in which case it means + emitting the correspondingly-named event (i.e., connect to self) + """ + self._sigs[signal]["callbacks"].append(slot) + + def _signal(self, event): + """This is called by a an action on a widget + + Within an self.ignore_events context, nothing happens. + + Tests can execute this method by directly changing the values of + widget components. + """ + if not self._ignoring_events: + wn = "-".join([event.obj.name, event.name]) + if wn in self._map and self._map[wn] in self._sigs: + self._emit(self._map[wn], event.new) + + @contextlib.contextmanager + def ignore_events(self): + """Temporarily turn off events processing in this instance + + (does not propagate to children) + """ + self._ignoring_events = True + try: + yield + finally: + self._ignoring_events = False + + def _emit(self, sig, value=None): + """An event happened, call its callbacks + + This method can be used in tests to simulate message passing without + directly changing visual elements. + + Calling of callbacks will halt whenever one returns False. + """ + logger.log(self._sigs[sig]["log"], f"{sig}: {value}") + for callback in self._sigs[sig]["callbacks"]: + if isinstance(callback, str): + self._emit(callback) + else: + try: + # running callbacks should not break the interface + ret = callback(value) + if ret is False: + break + except Exception as e: + logger.exception( + "Exception (%s) while executing callback for signal: %s", + e, + sig, + ) + + def show(self, threads=False): + """Open a new browser tab and display this instance's interface""" + self.panel.show(threads=threads, verbose=False) + return self + + +class SingleSelect(SigSlot): + """A multiselect which only allows you to select one item for an event""" + + signals = ["_selected", "selected"] # the first is internal + slots = ["set_options", "set_selection", "add", "clear", "select"] + + def __init__(self, **kwargs): + self.kwargs = kwargs + super().__init__() + + def _setup(self): + self.panel = pn.widgets.MultiSelect(**self.kwargs) + self._register(self.panel, "_selected", "value") + self._register(None, "selected") + self.connect("_selected", self.select_one) + + def _signal(self, *args, **kwargs): + super()._signal(*args, **kwargs) + + def select_one(self, *_): + with self.ignore_events(): + val = [self.panel.value[-1]] if self.panel.value else [] + self.panel.value = val + self._emit("selected", self.panel.value) + + def set_options(self, options): + self.panel.options = options + + def clear(self): + self.panel.options = [] + + @property + def value(self): + return self.panel.value + + def set_selection(self, selection): + self.panel.value = [selection] + + +class FileSelector(SigSlot): + """Panel-based graphical file selector widget + + Instances of this widget are interactive and can be displayed in jupyter by having + them as the output of a cell, or in a separate browser tab using ``.show()``. + """ + + signals = [ + "protocol_changed", + "selection_changed", + "directory_entered", + "home_clicked", + "up_clicked", + "go_clicked", + "filters_changed", + ] + slots = ["set_filters", "go_home"] + + def __init__(self, url=None, filters=None, ignore=None, kwargs=None): + """ + + Parameters + ---------- + url : str (optional) + Initial value of the URL to populate the dialog; should include protocol + filters : list(str) (optional) + File endings to include in the listings. If not included, all files are + allowed. Does not affect directories. + If given, the endings will appear as checkboxes in the interface + ignore : list(str) (optional) + Regex(s) of file basename patterns to ignore, e.g., "\\." for typical + hidden files on posix + kwargs : dict (optional) + To pass to file system instance + """ + if url: + self.init_protocol, url = split_protocol(url) + else: + self.init_protocol, url = "file", os.getcwd() + self.init_url = url + self.init_kwargs = (kwargs if isinstance(kwargs, str) else str(kwargs)) or "{}" + self.filters = filters + self.ignore = [re.compile(i) for i in ignore or []] + self._fs = None + super().__init__() + + def _setup(self): + self.url = pn.widgets.TextInput( + name="url", + value=self.init_url, + align="end", + sizing_mode="stretch_width", + width_policy="max", + ) + self.protocol = pn.widgets.Select( + options=sorted(known_implementations), + value=self.init_protocol, + name="protocol", + align="center", + ) + self.kwargs = pn.widgets.TextInput( + name="kwargs", value=self.init_kwargs, align="center" + ) + self.go = pn.widgets.Button(name="⇨", align="end", width=45) + self.main = SingleSelect(size=10) + self.home = pn.widgets.Button(name="🏠", width=40, height=30, align="end") + self.up = pn.widgets.Button(name="‹", width=30, height=30, align="end") + + self._register(self.protocol, "protocol_changed", auto=True) + self._register(self.go, "go_clicked", "clicks", auto=True) + self._register(self.up, "up_clicked", "clicks", auto=True) + self._register(self.home, "home_clicked", "clicks", auto=True) + self._register(None, "selection_changed") + self.main.connect("selected", self.selection_changed) + self._register(None, "directory_entered") + self.prev_protocol = self.protocol.value + self.prev_kwargs = self.storage_options + + self.filter_sel = pn.widgets.CheckBoxGroup( + value=[], options=[], inline=False, align="end", width_policy="min" + ) + self._register(self.filter_sel, "filters_changed", auto=True) + + self.panel = pn.Column( + pn.Row(self.protocol, self.kwargs), + pn.Row(self.home, self.up, self.url, self.go, self.filter_sel), + self.main.panel, + ) + self.set_filters(self.filters) + self.go_clicked() + + def set_filters(self, filters=None): + self.filters = filters + if filters: + self.filter_sel.options = filters + self.filter_sel.value = filters + else: + self.filter_sel.options = [] + self.filter_sel.value = [] + + @property + def storage_options(self): + """Value of the kwargs box as a dictionary""" + return ast.literal_eval(self.kwargs.value) or {} + + @property + def fs(self): + """Current filesystem instance""" + if self._fs is None: + cls = get_filesystem_class(self.protocol.value) + self._fs = cls(**self.storage_options) + return self._fs + + @property + def urlpath(self): + """URL of currently selected item""" + return ( + (f"{self.protocol.value}://{self.main.value[0]}") + if self.main.value + else None + ) + + def open_file(self, mode="rb", compression=None, encoding=None): + """Create OpenFile instance for the currently selected item + + For example, in a notebook you might do something like + + .. code-block:: + + [ ]: sel = FileSelector(); sel + + # user selects their file + + [ ]: with sel.open_file('rb') as f: + ... out = f.read() + + Parameters + ---------- + mode: str (optional) + Open mode for the file. + compression: str (optional) + The interact with the file as compressed. Set to 'infer' to guess + compression from the file ending + encoding: str (optional) + If using text mode, use this encoding; defaults to UTF8. + """ + if self.urlpath is None: + raise ValueError("No file selected") + return OpenFile(self.fs, self.urlpath, mode, compression, encoding) + + def filters_changed(self, values): + self.filters = values + self.go_clicked() + + def selection_changed(self, *_): + if self.urlpath is None: + return + if self.fs.isdir(self.urlpath): + self.url.value = self.fs._strip_protocol(self.urlpath) + self.go_clicked() + + def go_clicked(self, *_): + if ( + self.prev_protocol != self.protocol.value + or self.prev_kwargs != self.storage_options + ): + self._fs = None # causes fs to be recreated + self.prev_protocol = self.protocol.value + self.prev_kwargs = self.storage_options + listing = sorted( + self.fs.ls(self.url.value, detail=True), key=lambda x: x["name"] + ) + listing = [ + l + for l in listing + if not any(i.match(l["name"].rsplit("/", 1)[-1]) for i in self.ignore) + ] + folders = { + "📁 " + o["name"].rsplit("/", 1)[-1]: o["name"] + for o in listing + if o["type"] == "directory" + } + files = { + "📄 " + o["name"].rsplit("/", 1)[-1]: o["name"] + for o in listing + if o["type"] == "file" + } + if self.filters: + files = { + k: v + for k, v in files.items() + if any(v.endswith(ext) for ext in self.filters) + } + self.main.set_options(dict(**folders, **files)) + + def protocol_changed(self, *_): + self._fs = None + self.main.options = [] + self.url.value = "" + + def home_clicked(self, *_): + self.protocol.value = self.init_protocol + self.kwargs.value = self.init_kwargs + self.url.value = self.init_url + self.go_clicked() + + def up_clicked(self, *_): + self.url.value = self.fs._parent(self.url.value) + self.go_clicked() diff --git a/lib/python3.12/site-packages/fsspec/implementations/__init__.py b/lib/python3.12/site-packages/fsspec/implementations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0ed5681bc24e218fe9a642760988335ddd6ccf42 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/arrow.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/arrow.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..222156a5cc3c341d1c9aead7adaa7dc8798e8196 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/arrow.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/asyn_wrapper.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/asyn_wrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..60b48401966abca045860807f44f9c96702a32d9 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/asyn_wrapper.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cache_mapper.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cache_mapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8216ba4c7a16dbbe82ceb8c45a5b1175411f1dbf Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cache_mapper.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cache_metadata.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cache_metadata.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f71dc4d4f740eea282ac8deba4a081e744089bb6 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cache_metadata.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cached.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cached.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3f5faa498bf44cfe354d4114c7c5e53b92b03d46 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/cached.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dask.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dask.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a59f255d10a60835519dd13d3f554fd33f816313 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dask.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/data.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/data.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4c771d4e8e084331be8aaaa5449b74feb24fc9da Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/data.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dbfs.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dbfs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b48c84286fbb7328d7a43ca9d9f3eece436cd9eb Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dbfs.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dirfs.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dirfs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7905ac7274fbdbf5fccc65e9359f6f6babfe986 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/dirfs.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/ftp.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/ftp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62163ec81b12d1dcbd3bd412b3bdc3456379dfca Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/ftp.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/git.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/git.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5a8eb0375364ac63f9f86fcf90fa39d4c7321549 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/git.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/github.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/github.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a2cc732ed28541298657a5c99198c5b06d4a3cf7 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/github.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/http.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/http.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5946e74c6e64b532344a159d2fda0a6beb99c01c Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/http.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/http_sync.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/http_sync.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08aacf9d59f6c5dc45a54a8e07c06a321781e3dd Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/http_sync.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/jupyter.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/jupyter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3294fe992856eeeef633d0cbe384ff66d51187ec Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/jupyter.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/libarchive.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/libarchive.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f12d59f153faff0997bbe2363fe60f811aee952c Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/libarchive.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/local.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/local.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6a49dca0e2e6ff5149ff8a3f0b3d97f6a9e42c44 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/local.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/memory.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/memory.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1efe90c19f0a9becf675244fe4008acd7ae7e5d8 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/memory.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/reference.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/reference.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..056b8140bc95ba08ac1a94ed48c05ea88d4a1d84 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/reference.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/sftp.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/sftp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f581336bb116816ba8b2911b25eef71717e55f0c Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/sftp.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/smb.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/smb.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48bb1bba5df03441a9f7fa30a10ac777e498eb03 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/smb.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/tar.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/tar.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..872f92a15f149af0a74357542ee4f38c661c75a3 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/tar.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/webhdfs.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/webhdfs.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b34f51b2588082bad2e6d589e5bc62a57406b51f Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/webhdfs.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/__pycache__/zip.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/zip.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2886445f3d6423456a9ea13d8630e10606aeabf9 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/implementations/__pycache__/zip.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/implementations/arrow.py b/lib/python3.12/site-packages/fsspec/implementations/arrow.py new file mode 100644 index 0000000000000000000000000000000000000000..530df901a7225bab4afb9f08d06540bc18e91aef --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/arrow.py @@ -0,0 +1,304 @@ +import errno +import io +import os +import secrets +import shutil +from contextlib import suppress +from functools import cached_property, wraps +from urllib.parse import parse_qs + +from fsspec.spec import AbstractFileSystem +from fsspec.utils import ( + get_package_version_without_import, + infer_storage_options, + mirror_from, + tokenize, +) + + +def wrap_exceptions(func): + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except OSError as exception: + if not exception.args: + raise + + message, *args = exception.args + if isinstance(message, str) and "does not exist" in message: + raise FileNotFoundError(errno.ENOENT, message) from exception + else: + raise + + return wrapper + + +PYARROW_VERSION = None + + +class ArrowFSWrapper(AbstractFileSystem): + """FSSpec-compatible wrapper of pyarrow.fs.FileSystem. + + Parameters + ---------- + fs : pyarrow.fs.FileSystem + + """ + + root_marker = "/" + + def __init__(self, fs, **kwargs): + global PYARROW_VERSION + PYARROW_VERSION = get_package_version_without_import("pyarrow") + self.fs = fs + super().__init__(**kwargs) + + @property + def protocol(self): + return self.fs.type_name + + @cached_property + def fsid(self): + return "hdfs_" + tokenize(self.fs.host, self.fs.port) + + @classmethod + def _strip_protocol(cls, path): + ops = infer_storage_options(path) + path = ops["path"] + if path.startswith("//"): + # special case for "hdfs://path" (without the triple slash) + path = path[1:] + return path + + def ls(self, path, detail=False, **kwargs): + path = self._strip_protocol(path) + from pyarrow.fs import FileSelector + + entries = [ + self._make_entry(entry) + for entry in self.fs.get_file_info(FileSelector(path)) + ] + if detail: + return entries + else: + return [entry["name"] for entry in entries] + + def info(self, path, **kwargs): + path = self._strip_protocol(path) + [info] = self.fs.get_file_info([path]) + return self._make_entry(info) + + def exists(self, path): + path = self._strip_protocol(path) + try: + self.info(path) + except FileNotFoundError: + return False + else: + return True + + def _make_entry(self, info): + from pyarrow.fs import FileType + + if info.type is FileType.Directory: + kind = "directory" + elif info.type is FileType.File: + kind = "file" + elif info.type is FileType.NotFound: + raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), info.path) + else: + kind = "other" + + return { + "name": info.path, + "size": info.size, + "type": kind, + "mtime": info.mtime, + } + + @wrap_exceptions + def cp_file(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1).rstrip("/") + path2 = self._strip_protocol(path2).rstrip("/") + + with self._open(path1, "rb") as lstream: + tmp_fname = f"{path2}.tmp.{secrets.token_hex(6)}" + try: + with self.open(tmp_fname, "wb") as rstream: + shutil.copyfileobj(lstream, rstream) + self.fs.move(tmp_fname, path2) + except BaseException: + with suppress(FileNotFoundError): + self.fs.delete_file(tmp_fname) + raise + + @wrap_exceptions + def mv(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1).rstrip("/") + path2 = self._strip_protocol(path2).rstrip("/") + self.fs.move(path1, path2) + + @wrap_exceptions + def rm_file(self, path): + path = self._strip_protocol(path) + self.fs.delete_file(path) + + @wrap_exceptions + def rm(self, path, recursive=False, maxdepth=None): + path = self._strip_protocol(path).rstrip("/") + if self.isdir(path): + if recursive: + self.fs.delete_dir(path) + else: + raise ValueError("Can't delete directories without recursive=False") + else: + self.fs.delete_file(path) + + @wrap_exceptions + def _open(self, path, mode="rb", block_size=None, seekable=True, **kwargs): + if mode == "rb": + if seekable: + method = self.fs.open_input_file + else: + method = self.fs.open_input_stream + elif mode == "wb": + method = self.fs.open_output_stream + elif mode == "ab": + method = self.fs.open_append_stream + else: + raise ValueError(f"unsupported mode for Arrow filesystem: {mode!r}") + + _kwargs = {} + if mode != "rb" or not seekable: + if int(PYARROW_VERSION.split(".")[0]) >= 4: + # disable compression auto-detection + _kwargs["compression"] = None + stream = method(path, **_kwargs) + + return ArrowFile(self, stream, path, mode, block_size, **kwargs) + + @wrap_exceptions + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if create_parents: + self.makedirs(path, exist_ok=True) + else: + self.fs.create_dir(path, recursive=False) + + @wrap_exceptions + def makedirs(self, path, exist_ok=False): + path = self._strip_protocol(path) + self.fs.create_dir(path, recursive=True) + + @wrap_exceptions + def rmdir(self, path): + path = self._strip_protocol(path) + self.fs.delete_dir(path) + + @wrap_exceptions + def modified(self, path): + path = self._strip_protocol(path) + return self.fs.get_file_info(path).mtime + + def cat_file(self, path, start=None, end=None, **kwargs): + kwargs["seekable"] = start not in [None, 0] + return super().cat_file(path, start=None, end=None, **kwargs) + + def get_file(self, rpath, lpath, **kwargs): + kwargs["seekable"] = False + super().get_file(rpath, lpath, **kwargs) + + +@mirror_from( + "stream", + [ + "read", + "seek", + "tell", + "write", + "readable", + "writable", + "close", + "size", + "seekable", + ], +) +class ArrowFile(io.IOBase): + def __init__(self, fs, stream, path, mode, block_size=None, **kwargs): + self.path = path + self.mode = mode + + self.fs = fs + self.stream = stream + + self.blocksize = self.block_size = block_size + self.kwargs = kwargs + + def __enter__(self): + return self + + def __exit__(self, *args): + return self.close() + + +class HadoopFileSystem(ArrowFSWrapper): + """A wrapper on top of the pyarrow.fs.HadoopFileSystem + to connect it's interface with fsspec""" + + protocol = "hdfs" + + def __init__( + self, + host="default", + port=0, + user=None, + kerb_ticket=None, + replication=3, + extra_conf=None, + **kwargs, + ): + """ + + Parameters + ---------- + host: str + Hostname, IP or "default" to try to read from Hadoop config + port: int + Port to connect on, or default from Hadoop config if 0 + user: str or None + If given, connect as this username + kerb_ticket: str or None + If given, use this ticket for authentication + replication: int + set replication factor of file for write operations. default value is 3. + extra_conf: None or dict + Passed on to HadoopFileSystem + """ + from pyarrow.fs import HadoopFileSystem + + fs = HadoopFileSystem( + host=host, + port=port, + user=user, + kerb_ticket=kerb_ticket, + replication=replication, + extra_conf=extra_conf, + ) + super().__init__(fs=fs, **kwargs) + + @staticmethod + def _get_kwargs_from_urls(path): + ops = infer_storage_options(path) + out = {} + if ops.get("host", None): + out["host"] = ops["host"] + if ops.get("username", None): + out["user"] = ops["username"] + if ops.get("port", None): + out["port"] = ops["port"] + if ops.get("url_query", None): + queries = parse_qs(ops["url_query"]) + if queries.get("replication", None): + out["replication"] = int(queries["replication"][0]) + return out diff --git a/lib/python3.12/site-packages/fsspec/implementations/asyn_wrapper.py b/lib/python3.12/site-packages/fsspec/implementations/asyn_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..ee009f3bbcb5718e6c534f36f53ce4307999d54e --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/asyn_wrapper.py @@ -0,0 +1,103 @@ +import asyncio +import functools +import inspect + +from fsspec.asyn import AsyncFileSystem, running_async + + +def async_wrapper(func, obj=None): + """ + Wraps a synchronous function to make it awaitable. + + Parameters + ---------- + func : callable + The synchronous function to wrap. + obj : object, optional + The instance to bind the function to, if applicable. + + Returns + ------- + coroutine + An awaitable version of the function. + """ + + @functools.wraps(func) + async def wrapper(*args, **kwargs): + return await asyncio.to_thread(func, *args, **kwargs) + + return wrapper + + +class AsyncFileSystemWrapper(AsyncFileSystem): + """ + A wrapper class to convert a synchronous filesystem into an asynchronous one. + + This class takes an existing synchronous filesystem implementation and wraps all + its methods to provide an asynchronous interface. + + Parameters + ---------- + sync_fs : AbstractFileSystem + The synchronous filesystem instance to wrap. + """ + + protocol = "async_wrapper" + cachable = False + + def __init__(self, fs, *args, asynchronous=None, **kwargs): + if asynchronous is None: + asynchronous = running_async() + super().__init__(*args, asynchronous=asynchronous, **kwargs) + self.sync_fs = fs + self.protocol = self.sync_fs.protocol + self._wrap_all_sync_methods() + + @property + def fsid(self): + return f"async_{self.sync_fs.fsid}" + + def _wrap_all_sync_methods(self): + """ + Wrap all synchronous methods of the underlying filesystem with asynchronous versions. + """ + excluded_methods = {"open"} + for method_name in dir(self.sync_fs): + if method_name.startswith("_") or method_name in excluded_methods: + continue + + attr = inspect.getattr_static(self.sync_fs, method_name) + if isinstance(attr, property): + continue + + method = getattr(self.sync_fs, method_name) + if callable(method) and not asyncio.iscoroutinefunction(method): + async_method = async_wrapper(method, obj=self) + setattr(self, f"_{method_name}", async_method) + + @classmethod + def wrap_class(cls, sync_fs_class): + """ + Create a new class that can be used to instantiate an AsyncFileSystemWrapper + with lazy instantiation of the underlying synchronous filesystem. + + Parameters + ---------- + sync_fs_class : type + The class of the synchronous filesystem to wrap. + + Returns + ------- + type + A new class that wraps the provided synchronous filesystem class. + """ + + class GeneratedAsyncFileSystemWrapper(cls): + def __init__(self, *args, **kwargs): + sync_fs = sync_fs_class(*args, **kwargs) + super().__init__(sync_fs) + + GeneratedAsyncFileSystemWrapper.__name__ = ( + f"Async{sync_fs_class.__name__}Wrapper" + ) + return GeneratedAsyncFileSystemWrapper diff --git a/lib/python3.12/site-packages/fsspec/implementations/cache_mapper.py b/lib/python3.12/site-packages/fsspec/implementations/cache_mapper.py new file mode 100644 index 0000000000000000000000000000000000000000..6e7c7d88afdddf12f77b26bb635bd8bf1e2bd7f1 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/cache_mapper.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import abc +import hashlib + +from fsspec.implementations.local import make_path_posix + + +class AbstractCacheMapper(abc.ABC): + """Abstract super-class for mappers from remote URLs to local cached + basenames. + """ + + @abc.abstractmethod + def __call__(self, path: str) -> str: ... + + def __eq__(self, other: object) -> bool: + # Identity only depends on class. When derived classes have attributes + # they will need to be included. + return isinstance(other, type(self)) + + def __hash__(self) -> int: + # Identity only depends on class. When derived classes have attributes + # they will need to be included. + return hash(type(self)) + + +class BasenameCacheMapper(AbstractCacheMapper): + """Cache mapper that uses the basename of the remote URL and a fixed number + of directory levels above this. + + The default is zero directory levels, meaning different paths with the same + basename will have the same cached basename. + """ + + def __init__(self, directory_levels: int = 0): + if directory_levels < 0: + raise ValueError( + "BasenameCacheMapper requires zero or positive directory_levels" + ) + self.directory_levels = directory_levels + + # Separator for directories when encoded as strings. + self._separator = "_@_" + + def __call__(self, path: str) -> str: + path = make_path_posix(path) + prefix, *bits = path.rsplit("/", self.directory_levels + 1) + if bits: + return self._separator.join(bits) + else: + return prefix # No separator found, simple filename + + def __eq__(self, other: object) -> bool: + return super().__eq__(other) and self.directory_levels == other.directory_levels + + def __hash__(self) -> int: + return super().__hash__() ^ hash(self.directory_levels) + + +class HashCacheMapper(AbstractCacheMapper): + """Cache mapper that uses a hash of the remote URL.""" + + def __call__(self, path: str) -> str: + return hashlib.sha256(path.encode()).hexdigest() + + +def create_cache_mapper(same_names: bool) -> AbstractCacheMapper: + """Factory method to create cache mapper for backward compatibility with + ``CachingFileSystem`` constructor using ``same_names`` kwarg. + """ + if same_names: + return BasenameCacheMapper() + else: + return HashCacheMapper() diff --git a/lib/python3.12/site-packages/fsspec/implementations/cache_metadata.py b/lib/python3.12/site-packages/fsspec/implementations/cache_metadata.py new file mode 100644 index 0000000000000000000000000000000000000000..bd9b5cdd99d7f4a0a989c0f7d0c70ddcf324816a --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/cache_metadata.py @@ -0,0 +1,232 @@ +from __future__ import annotations + +import os +import pickle +import time +from typing import TYPE_CHECKING + +from fsspec.utils import atomic_write + +try: + import ujson as json +except ImportError: + if not TYPE_CHECKING: + import json + +if TYPE_CHECKING: + from typing import Any, Dict, Iterator, Literal + + from typing_extensions import TypeAlias + + from .cached import CachingFileSystem + + Detail: TypeAlias = Dict[str, Any] + + +class CacheMetadata: + """Cache metadata. + + All reading and writing of cache metadata is performed by this class, + accessing the cached files and blocks is not. + + Metadata is stored in a single file per storage directory in JSON format. + For backward compatibility, also reads metadata stored in pickle format + which is converted to JSON when next saved. + """ + + def __init__(self, storage: list[str]): + """ + + Parameters + ---------- + storage: list[str] + Directories containing cached files, must be at least one. Metadata + is stored in the last of these directories by convention. + """ + if not storage: + raise ValueError("CacheMetadata expects at least one storage location") + + self._storage = storage + self.cached_files: list[Detail] = [{}] + + # Private attribute to force saving of metadata in pickle format rather than + # JSON for use in tests to confirm can read both pickle and JSON formats. + self._force_save_pickle = False + + def _load(self, fn: str) -> Detail: + """Low-level function to load metadata from specific file""" + try: + with open(fn, "r") as f: + loaded = json.load(f) + except ValueError: + with open(fn, "rb") as f: + loaded = pickle.load(f) + for c in loaded.values(): + if isinstance(c.get("blocks"), list): + c["blocks"] = set(c["blocks"]) + return loaded + + def _save(self, metadata_to_save: Detail, fn: str) -> None: + """Low-level function to save metadata to specific file""" + if self._force_save_pickle: + with atomic_write(fn) as f: + pickle.dump(metadata_to_save, f) + else: + with atomic_write(fn, mode="w") as f: + json.dump(metadata_to_save, f) + + def _scan_locations( + self, writable_only: bool = False + ) -> Iterator[tuple[str, str, bool]]: + """Yield locations (filenames) where metadata is stored, and whether + writable or not. + + Parameters + ---------- + writable: bool + Set to True to only yield writable locations. + + Returns + ------- + Yields (str, str, bool) + """ + n = len(self._storage) + for i, storage in enumerate(self._storage): + writable = i == n - 1 + if writable_only and not writable: + continue + yield os.path.join(storage, "cache"), storage, writable + + def check_file( + self, path: str, cfs: CachingFileSystem | None + ) -> Literal[False] | tuple[Detail, str]: + """If path is in cache return its details, otherwise return ``False``. + + If the optional CachingFileSystem is specified then it is used to + perform extra checks to reject possible matches, such as if they are + too old. + """ + for (fn, base, _), cache in zip(self._scan_locations(), self.cached_files): + if path not in cache: + continue + detail = cache[path].copy() + + if cfs is not None: + if cfs.check_files and detail["uid"] != cfs.fs.ukey(path): + # Wrong file as determined by hash of file properties + continue + if cfs.expiry and time.time() - detail["time"] > cfs.expiry: + # Cached file has expired + continue + + fn = os.path.join(base, detail["fn"]) + if os.path.exists(fn): + return detail, fn + return False + + def clear_expired(self, expiry_time: int) -> tuple[list[str], bool]: + """Remove expired metadata from the cache. + + Returns names of files corresponding to expired metadata and a boolean + flag indicating whether the writable cache is empty. Caller is + responsible for deleting the expired files. + """ + expired_files = [] + for path, detail in self.cached_files[-1].copy().items(): + if time.time() - detail["time"] > expiry_time: + fn = detail.get("fn", "") + if not fn: + raise RuntimeError( + f"Cache metadata does not contain 'fn' for {path}" + ) + fn = os.path.join(self._storage[-1], fn) + expired_files.append(fn) + self.cached_files[-1].pop(path) + + if self.cached_files[-1]: + cache_path = os.path.join(self._storage[-1], "cache") + self._save(self.cached_files[-1], cache_path) + + writable_cache_empty = not self.cached_files[-1] + return expired_files, writable_cache_empty + + def load(self) -> None: + """Load all metadata from disk and store in ``self.cached_files``""" + cached_files = [] + for fn, _, _ in self._scan_locations(): + if os.path.exists(fn): + # TODO: consolidate blocks here + cached_files.append(self._load(fn)) + else: + cached_files.append({}) + self.cached_files = cached_files or [{}] + + def on_close_cached_file(self, f: Any, path: str) -> None: + """Perform side-effect actions on closing a cached file. + + The actual closing of the file is the responsibility of the caller. + """ + # File must be writeble, so in self.cached_files[-1] + c = self.cached_files[-1][path] + if c["blocks"] is not True and len(c["blocks"]) * f.blocksize >= f.size: + c["blocks"] = True + + def pop_file(self, path: str) -> str | None: + """Remove metadata of cached file. + + If path is in the cache, return the filename of the cached file, + otherwise return ``None``. Caller is responsible for deleting the + cached file. + """ + details = self.check_file(path, None) + if not details: + return None + _, fn = details + if fn.startswith(self._storage[-1]): + self.cached_files[-1].pop(path) + self.save() + else: + raise PermissionError( + "Can only delete cached file in last, writable cache location" + ) + return fn + + def save(self) -> None: + """Save metadata to disk""" + for (fn, _, writable), cache in zip(self._scan_locations(), self.cached_files): + if not writable: + continue + + if os.path.exists(fn): + cached_files = self._load(fn) + for k, c in cached_files.items(): + if k in cache: + if c["blocks"] is True or cache[k]["blocks"] is True: + c["blocks"] = True + else: + # self.cached_files[*][*]["blocks"] must continue to + # point to the same set object so that updates + # performed by MMapCache are propagated back to + # self.cached_files. + blocks = cache[k]["blocks"] + blocks.update(c["blocks"]) + c["blocks"] = blocks + c["time"] = max(c["time"], cache[k]["time"]) + c["uid"] = cache[k]["uid"] + + # Files can be added to cache after it was written once + for k, c in cache.items(): + if k not in cached_files: + cached_files[k] = c + else: + cached_files = cache + cache = {k: v.copy() for k, v in cached_files.items()} + for c in cache.values(): + if isinstance(c["blocks"], set): + c["blocks"] = list(c["blocks"]) + self._save(cache, fn) + self.cached_files[-1] = cached_files + + def update_file(self, path: str, detail: Detail) -> None: + """Update metadata for specific file in memory, do not save""" + self.cached_files[-1][path] = detail diff --git a/lib/python3.12/site-packages/fsspec/implementations/cached.py b/lib/python3.12/site-packages/fsspec/implementations/cached.py new file mode 100644 index 0000000000000000000000000000000000000000..8b8b832953a08ee2e64c11fb0f97eca48c7020ee --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/cached.py @@ -0,0 +1,929 @@ +from __future__ import annotations + +import inspect +import logging +import os +import tempfile +import time +import weakref +from shutil import rmtree +from typing import TYPE_CHECKING, Any, Callable, ClassVar + +from fsspec import AbstractFileSystem, filesystem +from fsspec.callbacks import DEFAULT_CALLBACK +from fsspec.compression import compr +from fsspec.core import BaseCache, MMapCache +from fsspec.exceptions import BlocksizeMismatchError +from fsspec.implementations.cache_mapper import create_cache_mapper +from fsspec.implementations.cache_metadata import CacheMetadata +from fsspec.spec import AbstractBufferedFile +from fsspec.transaction import Transaction +from fsspec.utils import infer_compression + +if TYPE_CHECKING: + from fsspec.implementations.cache_mapper import AbstractCacheMapper + +logger = logging.getLogger("fsspec.cached") + + +class WriteCachedTransaction(Transaction): + def complete(self, commit=True): + rpaths = [f.path for f in self.files] + lpaths = [f.fn for f in self.files] + if commit: + self.fs.put(lpaths, rpaths) + self.files.clear() + self.fs._intrans = False + self.fs._transaction = None + self.fs = None # break cycle + + +class CachingFileSystem(AbstractFileSystem): + """Locally caching filesystem, layer over any other FS + + This class implements chunk-wise local storage of remote files, for quick + access after the initial download. The files are stored in a given + directory with hashes of URLs for the filenames. If no directory is given, + a temporary one is used, which should be cleaned up by the OS after the + process ends. The files themselves are sparse (as implemented in + :class:`~fsspec.caching.MMapCache`), so only the data which is accessed + takes up space. + + Restrictions: + + - the block-size must be the same for each access of a given file, unless + all blocks of the file have already been read + - caching can only be applied to file-systems which produce files + derived from fsspec.spec.AbstractBufferedFile ; LocalFileSystem is also + allowed, for testing + """ + + protocol: ClassVar[str | tuple[str, ...]] = ("blockcache", "cached") + + def __init__( + self, + target_protocol=None, + cache_storage="TMP", + cache_check=10, + check_files=False, + expiry_time=604800, + target_options=None, + fs=None, + same_names: bool | None = None, + compression=None, + cache_mapper: AbstractCacheMapper | None = None, + **kwargs, + ): + """ + + Parameters + ---------- + target_protocol: str (optional) + Target filesystem protocol. Provide either this or ``fs``. + cache_storage: str or list(str) + Location to store files. If "TMP", this is a temporary directory, + and will be cleaned up by the OS when this process ends (or later). + If a list, each location will be tried in the order given, but + only the last will be considered writable. + cache_check: int + Number of seconds between reload of cache metadata + check_files: bool + Whether to explicitly see if the UID of the remote file matches + the stored one before using. Warning: some file systems such as + HTTP cannot reliably give a unique hash of the contents of some + path, so be sure to set this option to False. + expiry_time: int + The time in seconds after which a local copy is considered useless. + Set to falsy to prevent expiry. The default is equivalent to one + week. + target_options: dict or None + Passed to the instantiation of the FS, if fs is None. + fs: filesystem instance + The target filesystem to run against. Provide this or ``protocol``. + same_names: bool (optional) + By default, target URLs are hashed using a ``HashCacheMapper`` so + that files from different backends with the same basename do not + conflict. If this argument is ``true``, a ``BasenameCacheMapper`` + is used instead. Other cache mapper options are available by using + the ``cache_mapper`` keyword argument. Only one of this and + ``cache_mapper`` should be specified. + compression: str (optional) + To decompress on download. Can be 'infer' (guess from the URL name), + one of the entries in ``fsspec.compression.compr``, or None for no + decompression. + cache_mapper: AbstractCacheMapper (optional) + The object use to map from original filenames to cached filenames. + Only one of this and ``same_names`` should be specified. + """ + super().__init__(**kwargs) + if fs is None and target_protocol is None: + raise ValueError( + "Please provide filesystem instance(fs) or target_protocol" + ) + if not (fs is None) ^ (target_protocol is None): + raise ValueError( + "Both filesystems (fs) and target_protocol may not be both given." + ) + if cache_storage == "TMP": + tempdir = tempfile.mkdtemp() + storage = [tempdir] + weakref.finalize(self, self._remove_tempdir, tempdir) + else: + if isinstance(cache_storage, str): + storage = [cache_storage] + else: + storage = cache_storage + os.makedirs(storage[-1], exist_ok=True) + self.storage = storage + self.kwargs = target_options or {} + self.cache_check = cache_check + self.check_files = check_files + self.expiry = expiry_time + self.compression = compression + + # Size of cache in bytes. If None then the size is unknown and will be + # recalculated the next time cache_size() is called. On writes to the + # cache this is reset to None. + self._cache_size = None + + if same_names is not None and cache_mapper is not None: + raise ValueError( + "Cannot specify both same_names and cache_mapper in " + "CachingFileSystem.__init__" + ) + if cache_mapper is not None: + self._mapper = cache_mapper + else: + self._mapper = create_cache_mapper( + same_names if same_names is not None else False + ) + + self.target_protocol = ( + target_protocol + if isinstance(target_protocol, str) + else (fs.protocol if isinstance(fs.protocol, str) else fs.protocol[0]) + ) + self._metadata = CacheMetadata(self.storage) + self.load_cache() + self.fs = fs if fs is not None else filesystem(target_protocol, **self.kwargs) + + def _strip_protocol(path): + # acts as a method, since each instance has a difference target + return self.fs._strip_protocol(type(self)._strip_protocol(path)) + + self._strip_protocol: Callable = _strip_protocol + + @staticmethod + def _remove_tempdir(tempdir): + try: + rmtree(tempdir) + except Exception: + pass + + def _mkcache(self): + os.makedirs(self.storage[-1], exist_ok=True) + + def cache_size(self): + """Return size of cache in bytes. + + If more than one cache directory is in use, only the size of the last + one (the writable cache directory) is returned. + """ + if self._cache_size is None: + cache_dir = self.storage[-1] + self._cache_size = filesystem("file").du(cache_dir, withdirs=True) + return self._cache_size + + def load_cache(self): + """Read set of stored blocks from file""" + self._metadata.load() + self._mkcache() + self.last_cache = time.time() + + def save_cache(self): + """Save set of stored blocks from file""" + self._mkcache() + self._metadata.save() + self.last_cache = time.time() + self._cache_size = None + + def _check_cache(self): + """Reload caches if time elapsed or any disappeared""" + self._mkcache() + if not self.cache_check: + # explicitly told not to bother checking + return + timecond = time.time() - self.last_cache > self.cache_check + existcond = all(os.path.exists(storage) for storage in self.storage) + if timecond or not existcond: + self.load_cache() + + def _check_file(self, path): + """Is path in cache and still valid""" + path = self._strip_protocol(path) + self._check_cache() + return self._metadata.check_file(path, self) + + def clear_cache(self): + """Remove all files and metadata from the cache + + In the case of multiple cache locations, this clears only the last one, + which is assumed to be the read/write one. + """ + rmtree(self.storage[-1]) + self.load_cache() + self._cache_size = None + + def clear_expired_cache(self, expiry_time=None): + """Remove all expired files and metadata from the cache + + In the case of multiple cache locations, this clears only the last one, + which is assumed to be the read/write one. + + Parameters + ---------- + expiry_time: int + The time in seconds after which a local copy is considered useless. + If not defined the default is equivalent to the attribute from the + file caching instantiation. + """ + + if not expiry_time: + expiry_time = self.expiry + + self._check_cache() + + expired_files, writable_cache_empty = self._metadata.clear_expired(expiry_time) + for fn in expired_files: + if os.path.exists(fn): + os.remove(fn) + + if writable_cache_empty: + rmtree(self.storage[-1]) + self.load_cache() + + self._cache_size = None + + def pop_from_cache(self, path): + """Remove cached version of given file + + Deletes local copy of the given (remote) path. If it is found in a cache + location which is not the last, it is assumed to be read-only, and + raises PermissionError + """ + path = self._strip_protocol(path) + fn = self._metadata.pop_file(path) + if fn is not None: + os.remove(fn) + self._cache_size = None + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + """Wrap the target _open + + If the whole file exists in the cache, just open it locally and + return that. + + Otherwise, open the file on the target FS, and make it have a mmap + cache pointing to the location which we determine, in our cache. + The ``blocks`` instance is shared, so as the mmap cache instance + updates, so does the entry in our ``cached_files`` attribute. + We monkey-patch this file, so that when it closes, we call + ``close_and_update`` to save the state of the blocks. + """ + path = self._strip_protocol(path) + + path = self.fs._strip_protocol(path) + if "r" not in mode: + return self.fs._open( + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + **kwargs, + ) + detail = self._check_file(path) + if detail: + # file is in cache + detail, fn = detail + hash, blocks = detail["fn"], detail["blocks"] + if blocks is True: + # stored file is complete + logger.debug("Opening local copy of %s", path) + return open(fn, mode) + # TODO: action where partial file exists in read-only cache + logger.debug("Opening partially cached copy of %s", path) + else: + hash = self._mapper(path) + fn = os.path.join(self.storage[-1], hash) + blocks = set() + detail = { + "original": path, + "fn": hash, + "blocks": blocks, + "time": time.time(), + "uid": self.fs.ukey(path), + } + self._metadata.update_file(path, detail) + logger.debug("Creating local sparse file for %s", path) + + # call target filesystems open + self._mkcache() + f = self.fs._open( + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + cache_type="none", + **kwargs, + ) + if self.compression: + comp = ( + infer_compression(path) + if self.compression == "infer" + else self.compression + ) + f = compr[comp](f, mode="rb") + if "blocksize" in detail: + if detail["blocksize"] != f.blocksize: + raise BlocksizeMismatchError( + f"Cached file must be reopened with same block" + f" size as original (old: {detail['blocksize']}," + f" new {f.blocksize})" + ) + else: + detail["blocksize"] = f.blocksize + f.cache = MMapCache(f.blocksize, f._fetch_range, f.size, fn, blocks) + close = f.close + f.close = lambda: self.close_and_update(f, close) + self.save_cache() + return f + + def _parent(self, path): + return self.fs._parent(path) + + def hash_name(self, path: str, *args: Any) -> str: + # Kept for backward compatibility with downstream libraries. + # Ignores extra arguments, previously same_name boolean. + return self._mapper(path) + + def close_and_update(self, f, close): + """Called when a file is closing, so store the set of blocks""" + if f.closed: + return + path = self._strip_protocol(f.path) + self._metadata.on_close_cached_file(f, path) + try: + logger.debug("going to save") + self.save_cache() + logger.debug("saved") + except OSError: + logger.debug("Cache saving failed while closing file") + except NameError: + logger.debug("Cache save failed due to interpreter shutdown") + close() + f.closed = True + + def ls(self, path, detail=True): + return self.fs.ls(path, detail) + + def __getattribute__(self, item): + if item in { + "load_cache", + "_open", + "save_cache", + "close_and_update", + "__init__", + "__getattribute__", + "__reduce__", + "_make_local_details", + "open", + "cat", + "cat_file", + "cat_ranges", + "get", + "read_block", + "tail", + "head", + "info", + "ls", + "exists", + "isfile", + "isdir", + "_check_file", + "_check_cache", + "_mkcache", + "clear_cache", + "clear_expired_cache", + "pop_from_cache", + "local_file", + "_paths_from_path", + "get_mapper", + "open_many", + "commit_many", + "hash_name", + "__hash__", + "__eq__", + "to_json", + "to_dict", + "cache_size", + "pipe_file", + "pipe", + "start_transaction", + "end_transaction", + }: + # all the methods defined in this class. Note `open` here, since + # it calls `_open`, but is actually in superclass + return lambda *args, **kw: getattr(type(self), item).__get__(self)( + *args, **kw + ) + if item in ["__reduce_ex__"]: + raise AttributeError + if item in ["transaction"]: + # property + return type(self).transaction.__get__(self) + if item in ["_cache", "transaction_type"]: + # class attributes + return getattr(type(self), item) + if item == "__class__": + return type(self) + d = object.__getattribute__(self, "__dict__") + fs = d.get("fs", None) # fs is not immediately defined + if item in d: + return d[item] + elif fs is not None: + if item in fs.__dict__: + # attribute of instance + return fs.__dict__[item] + # attributed belonging to the target filesystem + cls = type(fs) + m = getattr(cls, item) + if (inspect.isfunction(m) or inspect.isdatadescriptor(m)) and ( + not hasattr(m, "__self__") or m.__self__ is None + ): + # instance method + return m.__get__(fs, cls) + return m # class method or attribute + else: + # attributes of the superclass, while target is being set up + return super().__getattribute__(item) + + def __eq__(self, other): + """Test for equality.""" + if self is other: + return True + if not isinstance(other, type(self)): + return False + return ( + self.storage == other.storage + and self.kwargs == other.kwargs + and self.cache_check == other.cache_check + and self.check_files == other.check_files + and self.expiry == other.expiry + and self.compression == other.compression + and self._mapper == other._mapper + and self.target_protocol == other.target_protocol + ) + + def __hash__(self): + """Calculate hash.""" + return ( + hash(tuple(self.storage)) + ^ hash(str(self.kwargs)) + ^ hash(self.cache_check) + ^ hash(self.check_files) + ^ hash(self.expiry) + ^ hash(self.compression) + ^ hash(self._mapper) + ^ hash(self.target_protocol) + ) + + +class WholeFileCacheFileSystem(CachingFileSystem): + """Caches whole remote files on first access + + This class is intended as a layer over any other file system, and + will make a local copy of each file accessed, so that all subsequent + reads are local. This is similar to ``CachingFileSystem``, but without + the block-wise functionality and so can work even when sparse files + are not allowed. See its docstring for definition of the init + arguments. + + The class still needs access to the remote store for listing files, + and may refresh cached files. + """ + + protocol = "filecache" + local_file = True + + def open_many(self, open_files, **kwargs): + paths = [of.path for of in open_files] + if "r" in open_files.mode: + self._mkcache() + else: + return [ + LocalTempFile( + self.fs, + path, + mode=open_files.mode, + fn=os.path.join(self.storage[-1], self._mapper(path)), + **kwargs, + ) + for path in paths + ] + + if self.compression: + raise NotImplementedError + details = [self._check_file(sp) for sp in paths] + downpath = [p for p, d in zip(paths, details) if not d] + downfn0 = [ + os.path.join(self.storage[-1], self._mapper(p)) + for p, d in zip(paths, details) + ] # keep these path names for opening later + downfn = [fn for fn, d in zip(downfn0, details) if not d] + if downpath: + # skip if all files are already cached and up to date + self.fs.get(downpath, downfn) + + # update metadata - only happens when downloads are successful + newdetail = [ + { + "original": path, + "fn": self._mapper(path), + "blocks": True, + "time": time.time(), + "uid": self.fs.ukey(path), + } + for path in downpath + ] + for path, detail in zip(downpath, newdetail): + self._metadata.update_file(path, detail) + self.save_cache() + + def firstpart(fn): + # helper to adapt both whole-file and simple-cache + return fn[1] if isinstance(fn, tuple) else fn + + return [ + open(firstpart(fn0) if fn0 else fn1, mode=open_files.mode) + for fn0, fn1 in zip(details, downfn0) + ] + + def commit_many(self, open_files): + self.fs.put([f.fn for f in open_files], [f.path for f in open_files]) + [f.close() for f in open_files] + for f in open_files: + # in case autocommit is off, and so close did not already delete + try: + os.remove(f.name) + except FileNotFoundError: + pass + self._cache_size = None + + def _make_local_details(self, path): + hash = self._mapper(path) + fn = os.path.join(self.storage[-1], hash) + detail = { + "original": path, + "fn": hash, + "blocks": True, + "time": time.time(), + "uid": self.fs.ukey(path), + } + self._metadata.update_file(path, detail) + logger.debug("Copying %s to local cache", path) + return fn + + def cat( + self, + path, + recursive=False, + on_error="raise", + callback=DEFAULT_CALLBACK, + **kwargs, + ): + paths = self.expand_path( + path, recursive=recursive, maxdepth=kwargs.get("maxdepth") + ) + getpaths = [] + storepaths = [] + fns = [] + out = {} + for p in paths.copy(): + try: + detail = self._check_file(p) + if not detail: + fn = self._make_local_details(p) + getpaths.append(p) + storepaths.append(fn) + else: + detail, fn = detail if isinstance(detail, tuple) else (None, detail) + fns.append(fn) + except Exception as e: + if on_error == "raise": + raise + if on_error == "return": + out[p] = e + paths.remove(p) + + if getpaths: + self.fs.get(getpaths, storepaths) + self.save_cache() + + callback.set_size(len(paths)) + for p, fn in zip(paths, fns): + with open(fn, "rb") as f: + out[p] = f.read() + callback.relative_update(1) + if isinstance(path, str) and len(paths) == 1 and recursive is False: + out = out[paths[0]] + return out + + def _open(self, path, mode="rb", **kwargs): + path = self._strip_protocol(path) + if "r" not in mode: + hash = self._mapper(path) + fn = os.path.join(self.storage[-1], hash) + user_specified_kwargs = { + k: v + for k, v in kwargs.items() + # those kwargs were added by open(), we don't want them + if k not in ["autocommit", "block_size", "cache_options"] + } + return LocalTempFile(self, path, mode=mode, fn=fn, **user_specified_kwargs) + detail = self._check_file(path) + if detail: + detail, fn = detail + _, blocks = detail["fn"], detail["blocks"] + if blocks is True: + logger.debug("Opening local copy of %s", path) + + # In order to support downstream filesystems to be able to + # infer the compression from the original filename, like + # the `TarFileSystem`, let's extend the `io.BufferedReader` + # fileobject protocol by adding a dedicated attribute + # `original`. + f = open(fn, mode) + f.original = detail.get("original") + return f + else: + raise ValueError( + f"Attempt to open partially cached file {path}" + f" as a wholly cached file" + ) + else: + fn = self._make_local_details(path) + kwargs["mode"] = mode + + # call target filesystems open + self._mkcache() + if self.compression: + with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2: + if isinstance(f, AbstractBufferedFile): + # want no type of caching if just downloading whole thing + f.cache = BaseCache(0, f.cache.fetcher, f.size) + comp = ( + infer_compression(path) + if self.compression == "infer" + else self.compression + ) + f = compr[comp](f, mode="rb") + data = True + while data: + block = getattr(f, "blocksize", 5 * 2**20) + data = f.read(block) + f2.write(data) + else: + self.fs.get_file(path, fn) + self.save_cache() + return self._open(path, mode) + + +class SimpleCacheFileSystem(WholeFileCacheFileSystem): + """Caches whole remote files on first access + + This class is intended as a layer over any other file system, and + will make a local copy of each file accessed, so that all subsequent + reads are local. This implementation only copies whole files, and + does not keep any metadata about the download time or file details. + It is therefore safer to use in multi-threaded/concurrent situations. + + This is the only of the caching filesystems that supports write: you will + be given a real local open file, and upon close and commit, it will be + uploaded to the target filesystem; the writability or the target URL is + not checked until that time. + + """ + + protocol = "simplecache" + local_file = True + transaction_type = WriteCachedTransaction + + def __init__(self, **kwargs): + kw = kwargs.copy() + for key in ["cache_check", "expiry_time", "check_files"]: + kw[key] = False + super().__init__(**kw) + for storage in self.storage: + if not os.path.exists(storage): + os.makedirs(storage, exist_ok=True) + + def _check_file(self, path): + self._check_cache() + sha = self._mapper(path) + for storage in self.storage: + fn = os.path.join(storage, sha) + if os.path.exists(fn): + return fn + + def save_cache(self): + pass + + def load_cache(self): + pass + + def pipe_file(self, path, value=None, **kwargs): + if self._intrans: + with self.open(path, "wb") as f: + f.write(value) + else: + super().pipe_file(path, value) + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + details = [] + try: + details = self.fs.ls( + path, detail=True, **kwargs + ).copy() # don't edit original! + except FileNotFoundError as e: + ex = e + else: + ex = None + if self._intrans: + path1 = path.rstrip("/") + "/" + for f in self.transaction.files: + if f.path == path: + details.append( + {"name": path, "size": f.size or f.tell(), "type": "file"} + ) + elif f.path.startswith(path1): + if f.path.count("/") == path1.count("/"): + details.append( + {"name": f.path, "size": f.size or f.tell(), "type": "file"} + ) + else: + dname = "/".join(f.path.split("/")[: path1.count("/") + 1]) + details.append({"name": dname, "size": 0, "type": "directory"}) + if ex is not None and not details: + raise ex + if detail: + return details + return sorted(_["name"] for _ in details) + + def info(self, path, **kwargs): + path = self._strip_protocol(path) + if self._intrans: + f = [_ for _ in self.transaction.files if _.path == path] + if f: + size = os.path.getsize(f[0].fn) if f[0].closed else f[0].tell() + return {"name": path, "size": size, "type": "file"} + f = any(_.path.startswith(path + "/") for _ in self.transaction.files) + if f: + return {"name": path, "size": 0, "type": "directory"} + return self.fs.info(path, **kwargs) + + def pipe(self, path, value=None, **kwargs): + if isinstance(path, str): + self.pipe_file(self._strip_protocol(path), value, **kwargs) + elif isinstance(path, dict): + for k, v in path.items(): + self.pipe_file(self._strip_protocol(k), v, **kwargs) + else: + raise ValueError("path must be str or dict") + + def cat_ranges( + self, paths, starts, ends, max_gap=None, on_error="return", **kwargs + ): + lpaths = [self._check_file(p) for p in paths] + rpaths = [p for l, p in zip(lpaths, paths) if l is False] + lpaths = [l for l, p in zip(lpaths, paths) if l is False] + self.fs.get(rpaths, lpaths) + return super().cat_ranges( + paths, starts, ends, max_gap=max_gap, on_error=on_error, **kwargs + ) + + def _open(self, path, mode="rb", **kwargs): + path = self._strip_protocol(path) + sha = self._mapper(path) + + if "r" not in mode: + fn = os.path.join(self.storage[-1], sha) + user_specified_kwargs = { + k: v + for k, v in kwargs.items() + if k not in ["autocommit", "block_size", "cache_options"] + } # those were added by open() + return LocalTempFile( + self, + path, + mode=mode, + autocommit=not self._intrans, + fn=fn, + **user_specified_kwargs, + ) + fn = self._check_file(path) + if fn: + return open(fn, mode) + + fn = os.path.join(self.storage[-1], sha) + logger.debug("Copying %s to local cache", path) + kwargs["mode"] = mode + + self._mkcache() + self._cache_size = None + if self.compression: + with self.fs._open(path, **kwargs) as f, open(fn, "wb") as f2: + if isinstance(f, AbstractBufferedFile): + # want no type of caching if just downloading whole thing + f.cache = BaseCache(0, f.cache.fetcher, f.size) + comp = ( + infer_compression(path) + if self.compression == "infer" + else self.compression + ) + f = compr[comp](f, mode="rb") + data = True + while data: + block = getattr(f, "blocksize", 5 * 2**20) + data = f.read(block) + f2.write(data) + else: + self.fs.get_file(path, fn) + return self._open(path, mode) + + +class LocalTempFile: + """A temporary local file, which will be uploaded on commit""" + + def __init__(self, fs, path, fn, mode="wb", autocommit=True, seek=0, **kwargs): + self.fn = fn + self.fh = open(fn, mode) + self.mode = mode + if seek: + self.fh.seek(seek) + self.path = path + self.size = None + self.fs = fs + self.closed = False + self.autocommit = autocommit + self.kwargs = kwargs + + def __reduce__(self): + # always open in r+b to allow continuing writing at a location + return ( + LocalTempFile, + (self.fs, self.path, self.fn, "r+b", self.autocommit, self.tell()), + ) + + def __enter__(self): + return self.fh + + def __exit__(self, exc_type, exc_val, exc_tb): + self.close() + + def close(self): + # self.size = self.fh.tell() + if self.closed: + return + self.fh.close() + self.closed = True + if self.autocommit: + self.commit() + + def discard(self): + self.fh.close() + os.remove(self.fn) + + def commit(self): + self.fs.put(self.fn, self.path, **self.kwargs) + # we do not delete local copy - it's still in the cache + + @property + def name(self): + return self.fn + + def __repr__(self) -> str: + return f"LocalTempFile: {self.path}" + + def __getattr__(self, item): + return getattr(self.fh, item) diff --git a/lib/python3.12/site-packages/fsspec/implementations/dask.py b/lib/python3.12/site-packages/fsspec/implementations/dask.py new file mode 100644 index 0000000000000000000000000000000000000000..3e1276463db6866665e6a0fe114efc247971b57e --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/dask.py @@ -0,0 +1,152 @@ +import dask +from distributed.client import Client, _get_global_client +from distributed.worker import Worker + +from fsspec import filesystem +from fsspec.spec import AbstractBufferedFile, AbstractFileSystem +from fsspec.utils import infer_storage_options + + +def _get_client(client): + if client is None: + return _get_global_client() + elif isinstance(client, Client): + return client + else: + # e.g., connection string + return Client(client) + + +def _in_worker(): + return bool(Worker._instances) + + +class DaskWorkerFileSystem(AbstractFileSystem): + """View files accessible to a worker as any other remote file-system + + When instances are run on the worker, uses the real filesystem. When + run on the client, they call the worker to provide information or data. + + **Warning** this implementation is experimental, and read-only for now. + """ + + def __init__( + self, target_protocol=None, target_options=None, fs=None, client=None, **kwargs + ): + super().__init__(**kwargs) + if not (fs is None) ^ (target_protocol is None): + raise ValueError( + "Please provide one of filesystem instance (fs) or" + " target_protocol, not both" + ) + self.target_protocol = target_protocol + self.target_options = target_options + self.worker = None + self.client = client + self.fs = fs + self._determine_worker() + + @staticmethod + def _get_kwargs_from_urls(path): + so = infer_storage_options(path) + if "host" in so and "port" in so: + return {"client": f"{so['host']}:{so['port']}"} + else: + return {} + + def _determine_worker(self): + if _in_worker(): + self.worker = True + if self.fs is None: + self.fs = filesystem( + self.target_protocol, **(self.target_options or {}) + ) + else: + self.worker = False + self.client = _get_client(self.client) + self.rfs = dask.delayed(self) + + def mkdir(self, *args, **kwargs): + if self.worker: + self.fs.mkdir(*args, **kwargs) + else: + self.rfs.mkdir(*args, **kwargs).compute() + + def rm(self, *args, **kwargs): + if self.worker: + self.fs.rm(*args, **kwargs) + else: + self.rfs.rm(*args, **kwargs).compute() + + def copy(self, *args, **kwargs): + if self.worker: + self.fs.copy(*args, **kwargs) + else: + self.rfs.copy(*args, **kwargs).compute() + + def mv(self, *args, **kwargs): + if self.worker: + self.fs.mv(*args, **kwargs) + else: + self.rfs.mv(*args, **kwargs).compute() + + def ls(self, *args, **kwargs): + if self.worker: + return self.fs.ls(*args, **kwargs) + else: + return self.rfs.ls(*args, **kwargs).compute() + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + if self.worker: + return self.fs._open( + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + **kwargs, + ) + else: + return DaskFile( + fs=self, + path=path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_options=cache_options, + **kwargs, + ) + + def fetch_range(self, path, mode, start, end): + if self.worker: + with self._open(path, mode) as f: + f.seek(start) + return f.read(end - start) + else: + return self.rfs.fetch_range(path, mode, start, end).compute() + + +class DaskFile(AbstractBufferedFile): + def __init__(self, mode="rb", **kwargs): + if mode != "rb": + raise ValueError('Remote dask files can only be opened in "rb" mode') + super().__init__(**kwargs) + + def _upload_chunk(self, final=False): + pass + + def _initiate_upload(self): + """Create remote file/upload""" + pass + + def _fetch_range(self, start, end): + """Get the specified set of bytes from remote""" + return self.fs.fetch_range(self.path, self.mode, start, end) diff --git a/lib/python3.12/site-packages/fsspec/implementations/data.py b/lib/python3.12/site-packages/fsspec/implementations/data.py new file mode 100644 index 0000000000000000000000000000000000000000..519032305bed633f2ba8a6148076433caf81710b --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/data.py @@ -0,0 +1,58 @@ +import base64 +import io +from typing import Optional +from urllib.parse import unquote + +from fsspec import AbstractFileSystem + + +class DataFileSystem(AbstractFileSystem): + """A handy decoder for data-URLs + + Example + ------- + >>> with fsspec.open("data:,Hello%2C%20World%21") as f: + ... print(f.read()) + b"Hello, World!" + + See https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/Data_URLs + """ + + protocol = "data" + + def __init__(self, **kwargs): + """No parameters for this filesystem""" + super().__init__(**kwargs) + + def cat_file(self, path, start=None, end=None, **kwargs): + pref, data = path.split(",", 1) + if pref.endswith("base64"): + return base64.b64decode(data)[start:end] + return unquote(data).encode()[start:end] + + def info(self, path, **kwargs): + pref, name = path.split(",", 1) + data = self.cat_file(path) + mime = pref.split(":", 1)[1].split(";", 1)[0] + return {"name": name, "size": len(data), "type": "file", "mimetype": mime} + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + if "r" not in mode: + raise ValueError("Read only filesystem") + return io.BytesIO(self.cat_file(path)) + + @staticmethod + def encode(data: bytes, mime: Optional[str] = None): + """Format the given data into data-URL syntax + + This version always base64 encodes, even when the data is ascii/url-safe. + """ + return f"data:{mime or ''};base64,{base64.b64encode(data).decode()}" diff --git a/lib/python3.12/site-packages/fsspec/implementations/dbfs.py b/lib/python3.12/site-packages/fsspec/implementations/dbfs.py new file mode 100644 index 0000000000000000000000000000000000000000..fa71e1a22c18ef77b4bd7c0fd651d92966db14af --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/dbfs.py @@ -0,0 +1,467 @@ +import base64 +import urllib + +import requests +import requests.exceptions +from requests.adapters import HTTPAdapter, Retry + +from fsspec import AbstractFileSystem +from fsspec.spec import AbstractBufferedFile + + +class DatabricksException(Exception): + """ + Helper class for exceptions raised in this module. + """ + + def __init__(self, error_code, message): + """Create a new DatabricksException""" + super().__init__(message) + + self.error_code = error_code + self.message = message + + +class DatabricksFileSystem(AbstractFileSystem): + """ + Get access to the Databricks filesystem implementation over HTTP. + Can be used inside and outside of a databricks cluster. + """ + + def __init__(self, instance, token, **kwargs): + """ + Create a new DatabricksFileSystem. + + Parameters + ---------- + instance: str + The instance URL of the databricks cluster. + For example for an Azure databricks cluster, this + has the form adb-..azuredatabricks.net. + token: str + Your personal token. Find out more + here: https://docs.databricks.com/dev-tools/api/latest/authentication.html + """ + self.instance = instance + self.token = token + self.session = requests.Session() + self.retries = Retry( + total=10, + backoff_factor=0.05, + status_forcelist=[408, 429, 500, 502, 503, 504], + ) + + self.session.mount("https://", HTTPAdapter(max_retries=self.retries)) + self.session.headers.update({"Authorization": f"Bearer {self.token}"}) + + super().__init__(**kwargs) + + def ls(self, path, detail=True, **kwargs): + """ + List the contents of the given path. + + Parameters + ---------- + path: str + Absolute path + detail: bool + Return not only the list of filenames, + but also additional information on file sizes + and types. + """ + out = self._ls_from_cache(path) + if not out: + try: + r = self._send_to_api( + method="get", endpoint="list", json={"path": path} + ) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + + raise + files = r["files"] + out = [ + { + "name": o["path"], + "type": "directory" if o["is_dir"] else "file", + "size": o["file_size"], + } + for o in files + ] + self.dircache[path] = out + + if detail: + return out + return [o["name"] for o in out] + + def makedirs(self, path, exist_ok=True): + """ + Create a given absolute path and all of its parents. + + Parameters + ---------- + path: str + Absolute path to create + exist_ok: bool + If false, checks if the folder + exists before creating it (and raises an + Exception if this is the case) + """ + if not exist_ok: + try: + # If the following succeeds, the path is already present + self._send_to_api( + method="get", endpoint="get-status", json={"path": path} + ) + raise FileExistsError(f"Path {path} already exists") + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + pass + + try: + self._send_to_api(method="post", endpoint="mkdirs", json={"path": path}) + except DatabricksException as e: + if e.error_code == "RESOURCE_ALREADY_EXISTS": + raise FileExistsError(e.message) from e + + raise + self.invalidate_cache(self._parent(path)) + + def mkdir(self, path, create_parents=True, **kwargs): + """ + Create a given absolute path and all of its parents. + + Parameters + ---------- + path: str + Absolute path to create + create_parents: bool + Whether to create all parents or not. + "False" is not implemented so far. + """ + if not create_parents: + raise NotImplementedError + + self.mkdirs(path, **kwargs) + + def rm(self, path, recursive=False, **kwargs): + """ + Remove the file or folder at the given absolute path. + + Parameters + ---------- + path: str + Absolute path what to remove + recursive: bool + Recursively delete all files in a folder. + """ + try: + self._send_to_api( + method="post", + endpoint="delete", + json={"path": path, "recursive": recursive}, + ) + except DatabricksException as e: + # This is not really an exception, it just means + # not everything was deleted so far + if e.error_code == "PARTIAL_DELETE": + self.rm(path=path, recursive=recursive) + elif e.error_code == "IO_ERROR": + # Using the same exception as the os module would use here + raise OSError(e.message) from e + + raise + self.invalidate_cache(self._parent(path)) + + def mv( + self, source_path, destination_path, recursive=False, maxdepth=None, **kwargs + ): + """ + Move a source to a destination path. + + A note from the original [databricks API manual] + (https://docs.databricks.com/dev-tools/api/latest/dbfs.html#move). + + When moving a large number of files the API call will time out after + approximately 60s, potentially resulting in partially moved data. + Therefore, for operations that move more than 10k files, we strongly + discourage using the DBFS REST API. + + Parameters + ---------- + source_path: str + From where to move (absolute path) + destination_path: str + To where to move (absolute path) + recursive: bool + Not implemented to far. + maxdepth: + Not implemented to far. + """ + if recursive: + raise NotImplementedError + if maxdepth: + raise NotImplementedError + + try: + self._send_to_api( + method="post", + endpoint="move", + json={"source_path": source_path, "destination_path": destination_path}, + ) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + elif e.error_code == "RESOURCE_ALREADY_EXISTS": + raise FileExistsError(e.message) from e + + raise + self.invalidate_cache(self._parent(source_path)) + self.invalidate_cache(self._parent(destination_path)) + + def _open(self, path, mode="rb", block_size="default", **kwargs): + """ + Overwrite the base class method to make sure to create a DBFile. + All arguments are copied from the base method. + + Only the default blocksize is allowed. + """ + return DatabricksFile(self, path, mode=mode, block_size=block_size, **kwargs) + + def _send_to_api(self, method, endpoint, json): + """ + Send the given json to the DBFS API + using a get or post request (specified by the argument `method`). + + Parameters + ---------- + method: str + Which http method to use for communication; "get" or "post". + endpoint: str + Where to send the request to (last part of the API URL) + json: dict + Dictionary of information to send + """ + if method == "post": + session_call = self.session.post + elif method == "get": + session_call = self.session.get + else: + raise ValueError(f"Do not understand method {method}") + + url = urllib.parse.urljoin(f"https://{self.instance}/api/2.0/dbfs/", endpoint) + + r = session_call(url, json=json) + + # The DBFS API will return a json, also in case of an exception. + # We want to preserve this information as good as possible. + try: + r.raise_for_status() + except requests.HTTPError as e: + # try to extract json error message + # if that fails, fall back to the original exception + try: + exception_json = e.response.json() + except Exception: + raise e from None + + raise DatabricksException(**exception_json) from e + + return r.json() + + def _create_handle(self, path, overwrite=True): + """ + Internal function to create a handle, which can be used to + write blocks of a file to DBFS. + A handle has a unique identifier which needs to be passed + whenever written during this transaction. + The handle is active for 10 minutes - after that a new + write transaction needs to be created. + Make sure to close the handle after you are finished. + + Parameters + ---------- + path: str + Absolute path for this file. + overwrite: bool + If a file already exist at this location, either overwrite + it or raise an exception. + """ + try: + r = self._send_to_api( + method="post", + endpoint="create", + json={"path": path, "overwrite": overwrite}, + ) + return r["handle"] + except DatabricksException as e: + if e.error_code == "RESOURCE_ALREADY_EXISTS": + raise FileExistsError(e.message) from e + + raise + + def _close_handle(self, handle): + """ + Close a handle, which was opened by :func:`_create_handle`. + + Parameters + ---------- + handle: str + Which handle to close. + """ + try: + self._send_to_api(method="post", endpoint="close", json={"handle": handle}) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + + raise + + def _add_data(self, handle, data): + """ + Upload data to an already opened file handle + (opened by :func:`_create_handle`). + The maximal allowed data size is 1MB after + conversion to base64. + Remember to close the handle when you are finished. + + Parameters + ---------- + handle: str + Which handle to upload data to. + data: bytes + Block of data to add to the handle. + """ + data = base64.b64encode(data).decode() + try: + self._send_to_api( + method="post", + endpoint="add-block", + json={"handle": handle, "data": data}, + ) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + elif e.error_code == "MAX_BLOCK_SIZE_EXCEEDED": + raise ValueError(e.message) from e + + raise + + def _get_data(self, path, start, end): + """ + Download data in bytes from a given absolute path in a block + from [start, start+length]. + The maximum number of allowed bytes to read is 1MB. + + Parameters + ---------- + path: str + Absolute path to download data from + start: int + Start position of the block + end: int + End position of the block + """ + try: + r = self._send_to_api( + method="get", + endpoint="read", + json={"path": path, "offset": start, "length": end - start}, + ) + return base64.b64decode(r["data"]) + except DatabricksException as e: + if e.error_code == "RESOURCE_DOES_NOT_EXIST": + raise FileNotFoundError(e.message) from e + elif e.error_code in ["INVALID_PARAMETER_VALUE", "MAX_READ_SIZE_EXCEEDED"]: + raise ValueError(e.message) from e + + raise + + def invalidate_cache(self, path=None): + if path is None: + self.dircache.clear() + else: + self.dircache.pop(path, None) + super().invalidate_cache(path) + + +class DatabricksFile(AbstractBufferedFile): + """ + Helper class for files referenced in the DatabricksFileSystem. + """ + + DEFAULT_BLOCK_SIZE = 1 * 2**20 # only allowed block size + + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + **kwargs, + ): + """ + Create a new instance of the DatabricksFile. + + The blocksize needs to be the default one. + """ + if block_size is None or block_size == "default": + block_size = self.DEFAULT_BLOCK_SIZE + + assert block_size == self.DEFAULT_BLOCK_SIZE, ( + f"Only the default block size is allowed, not {block_size}" + ) + + super().__init__( + fs, + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_type=cache_type, + cache_options=cache_options or {}, + **kwargs, + ) + + def _initiate_upload(self): + """Internal function to start a file upload""" + self.handle = self.fs._create_handle(self.path) + + def _upload_chunk(self, final=False): + """Internal function to add a chunk of data to a started upload""" + self.buffer.seek(0) + data = self.buffer.getvalue() + + data_chunks = [ + data[start:end] for start, end in self._to_sized_blocks(len(data)) + ] + + for data_chunk in data_chunks: + self.fs._add_data(handle=self.handle, data=data_chunk) + + if final: + self.fs._close_handle(handle=self.handle) + return True + + def _fetch_range(self, start, end): + """Internal function to download a block of data""" + return_buffer = b"" + length = end - start + for chunk_start, chunk_end in self._to_sized_blocks(length, start): + return_buffer += self.fs._get_data( + path=self.path, start=chunk_start, end=chunk_end + ) + + return return_buffer + + def _to_sized_blocks(self, length, start=0): + """Helper function to split a range from 0 to total_length into bloksizes""" + end = start + length + for data_chunk in range(start, end, self.blocksize): + data_start = data_chunk + data_end = min(end, data_chunk + self.blocksize) + yield data_start, data_end diff --git a/lib/python3.12/site-packages/fsspec/implementations/dirfs.py b/lib/python3.12/site-packages/fsspec/implementations/dirfs.py new file mode 100644 index 0000000000000000000000000000000000000000..c0623b82fc61ed4780fdbc1d69680f2b96f2b9f3 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/dirfs.py @@ -0,0 +1,388 @@ +from .. import filesystem +from ..asyn import AsyncFileSystem + + +class DirFileSystem(AsyncFileSystem): + """Directory prefix filesystem + + The DirFileSystem is a filesystem-wrapper. It assumes every path it is dealing with + is relative to the `path`. After performing the necessary paths operation it + delegates everything to the wrapped filesystem. + """ + + protocol = "dir" + + def __init__( + self, + path=None, + fs=None, + fo=None, + target_protocol=None, + target_options=None, + **storage_options, + ): + """ + Parameters + ---------- + path: str + Path to the directory. + fs: AbstractFileSystem + An instantiated filesystem to wrap. + target_protocol, target_options: + if fs is none, construct it from these + fo: str + Alternate for path; do not provide both + """ + super().__init__(**storage_options) + if fs is None: + fs = filesystem(protocol=target_protocol, **(target_options or {})) + path = path or fo + + if self.asynchronous and not fs.async_impl: + raise ValueError("can't use asynchronous with non-async fs") + + if fs.async_impl and self.asynchronous != fs.asynchronous: + raise ValueError("both dirfs and fs should be in the same sync/async mode") + + self.path = fs._strip_protocol(path) + self.fs = fs + + def _join(self, path): + if isinstance(path, str): + if not self.path: + return path + if not path: + return self.path + return self.fs.sep.join((self.path, self._strip_protocol(path))) + if isinstance(path, dict): + return {self._join(_path): value for _path, value in path.items()} + return [self._join(_path) for _path in path] + + def _relpath(self, path): + if isinstance(path, str): + if not self.path: + return path + # We need to account for S3FileSystem returning paths that do not + # start with a '/' + if path == self.path or ( + self.path.startswith(self.fs.sep) and path == self.path[1:] + ): + return "" + prefix = self.path + self.fs.sep + if self.path.startswith(self.fs.sep) and not path.startswith(self.fs.sep): + prefix = prefix[1:] + assert path.startswith(prefix) + return path[len(prefix) :] + return [self._relpath(_path) for _path in path] + + # Wrappers below + + @property + def sep(self): + return self.fs.sep + + async def set_session(self, *args, **kwargs): + return await self.fs.set_session(*args, **kwargs) + + async def _rm_file(self, path, **kwargs): + return await self.fs._rm_file(self._join(path), **kwargs) + + def rm_file(self, path, **kwargs): + return self.fs.rm_file(self._join(path), **kwargs) + + async def _rm(self, path, *args, **kwargs): + return await self.fs._rm(self._join(path), *args, **kwargs) + + def rm(self, path, *args, **kwargs): + return self.fs.rm(self._join(path), *args, **kwargs) + + async def _cp_file(self, path1, path2, **kwargs): + return await self.fs._cp_file(self._join(path1), self._join(path2), **kwargs) + + def cp_file(self, path1, path2, **kwargs): + return self.fs.cp_file(self._join(path1), self._join(path2), **kwargs) + + async def _copy( + self, + path1, + path2, + *args, + **kwargs, + ): + return await self.fs._copy( + self._join(path1), + self._join(path2), + *args, + **kwargs, + ) + + def copy(self, path1, path2, *args, **kwargs): + return self.fs.copy( + self._join(path1), + self._join(path2), + *args, + **kwargs, + ) + + async def _pipe(self, path, *args, **kwargs): + return await self.fs._pipe(self._join(path), *args, **kwargs) + + def pipe(self, path, *args, **kwargs): + return self.fs.pipe(self._join(path), *args, **kwargs) + + async def _pipe_file(self, path, *args, **kwargs): + return await self.fs._pipe_file(self._join(path), *args, **kwargs) + + def pipe_file(self, path, *args, **kwargs): + return self.fs.pipe_file(self._join(path), *args, **kwargs) + + async def _cat_file(self, path, *args, **kwargs): + return await self.fs._cat_file(self._join(path), *args, **kwargs) + + def cat_file(self, path, *args, **kwargs): + return self.fs.cat_file(self._join(path), *args, **kwargs) + + async def _cat(self, path, *args, **kwargs): + ret = await self.fs._cat( + self._join(path), + *args, + **kwargs, + ) + + if isinstance(ret, dict): + return {self._relpath(key): value for key, value in ret.items()} + + return ret + + def cat(self, path, *args, **kwargs): + ret = self.fs.cat( + self._join(path), + *args, + **kwargs, + ) + + if isinstance(ret, dict): + return {self._relpath(key): value for key, value in ret.items()} + + return ret + + async def _put_file(self, lpath, rpath, **kwargs): + return await self.fs._put_file(lpath, self._join(rpath), **kwargs) + + def put_file(self, lpath, rpath, **kwargs): + return self.fs.put_file(lpath, self._join(rpath), **kwargs) + + async def _put( + self, + lpath, + rpath, + *args, + **kwargs, + ): + return await self.fs._put( + lpath, + self._join(rpath), + *args, + **kwargs, + ) + + def put(self, lpath, rpath, *args, **kwargs): + return self.fs.put( + lpath, + self._join(rpath), + *args, + **kwargs, + ) + + async def _get_file(self, rpath, lpath, **kwargs): + return await self.fs._get_file(self._join(rpath), lpath, **kwargs) + + def get_file(self, rpath, lpath, **kwargs): + return self.fs.get_file(self._join(rpath), lpath, **kwargs) + + async def _get(self, rpath, *args, **kwargs): + return await self.fs._get(self._join(rpath), *args, **kwargs) + + def get(self, rpath, *args, **kwargs): + return self.fs.get(self._join(rpath), *args, **kwargs) + + async def _isfile(self, path): + return await self.fs._isfile(self._join(path)) + + def isfile(self, path): + return self.fs.isfile(self._join(path)) + + async def _isdir(self, path): + return await self.fs._isdir(self._join(path)) + + def isdir(self, path): + return self.fs.isdir(self._join(path)) + + async def _size(self, path): + return await self.fs._size(self._join(path)) + + def size(self, path): + return self.fs.size(self._join(path)) + + async def _exists(self, path): + return await self.fs._exists(self._join(path)) + + def exists(self, path): + return self.fs.exists(self._join(path)) + + async def _info(self, path, **kwargs): + info = await self.fs._info(self._join(path), **kwargs) + info = info.copy() + info["name"] = self._relpath(info["name"]) + return info + + def info(self, path, **kwargs): + info = self.fs.info(self._join(path), **kwargs) + info = info.copy() + info["name"] = self._relpath(info["name"]) + return info + + async def _ls(self, path, detail=True, **kwargs): + ret = (await self.fs._ls(self._join(path), detail=detail, **kwargs)).copy() + if detail: + out = [] + for entry in ret: + entry = entry.copy() + entry["name"] = self._relpath(entry["name"]) + out.append(entry) + return out + + return self._relpath(ret) + + def ls(self, path, detail=True, **kwargs): + ret = self.fs.ls(self._join(path), detail=detail, **kwargs).copy() + if detail: + out = [] + for entry in ret: + entry = entry.copy() + entry["name"] = self._relpath(entry["name"]) + out.append(entry) + return out + + return self._relpath(ret) + + async def _walk(self, path, *args, **kwargs): + async for root, dirs, files in self.fs._walk(self._join(path), *args, **kwargs): + yield self._relpath(root), dirs, files + + def walk(self, path, *args, **kwargs): + for root, dirs, files in self.fs.walk(self._join(path), *args, **kwargs): + yield self._relpath(root), dirs, files + + async def _glob(self, path, **kwargs): + detail = kwargs.get("detail", False) + ret = await self.fs._glob(self._join(path), **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + def glob(self, path, **kwargs): + detail = kwargs.get("detail", False) + ret = self.fs.glob(self._join(path), **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + async def _du(self, path, *args, **kwargs): + total = kwargs.get("total", True) + ret = await self.fs._du(self._join(path), *args, **kwargs) + if total: + return ret + + return {self._relpath(path): size for path, size in ret.items()} + + def du(self, path, *args, **kwargs): + total = kwargs.get("total", True) + ret = self.fs.du(self._join(path), *args, **kwargs) + if total: + return ret + + return {self._relpath(path): size for path, size in ret.items()} + + async def _find(self, path, *args, **kwargs): + detail = kwargs.get("detail", False) + ret = await self.fs._find(self._join(path), *args, **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + def find(self, path, *args, **kwargs): + detail = kwargs.get("detail", False) + ret = self.fs.find(self._join(path), *args, **kwargs) + if detail: + return {self._relpath(path): info for path, info in ret.items()} + return self._relpath(ret) + + async def _expand_path(self, path, *args, **kwargs): + return self._relpath( + await self.fs._expand_path(self._join(path), *args, **kwargs) + ) + + def expand_path(self, path, *args, **kwargs): + return self._relpath(self.fs.expand_path(self._join(path), *args, **kwargs)) + + async def _mkdir(self, path, *args, **kwargs): + return await self.fs._mkdir(self._join(path), *args, **kwargs) + + def mkdir(self, path, *args, **kwargs): + return self.fs.mkdir(self._join(path), *args, **kwargs) + + async def _makedirs(self, path, *args, **kwargs): + return await self.fs._makedirs(self._join(path), *args, **kwargs) + + def makedirs(self, path, *args, **kwargs): + return self.fs.makedirs(self._join(path), *args, **kwargs) + + def rmdir(self, path): + return self.fs.rmdir(self._join(path)) + + def mv(self, path1, path2, **kwargs): + return self.fs.mv( + self._join(path1), + self._join(path2), + **kwargs, + ) + + def touch(self, path, **kwargs): + return self.fs.touch(self._join(path), **kwargs) + + def created(self, path): + return self.fs.created(self._join(path)) + + def modified(self, path): + return self.fs.modified(self._join(path)) + + def sign(self, path, *args, **kwargs): + return self.fs.sign(self._join(path), *args, **kwargs) + + def __repr__(self): + return f"{self.__class__.__qualname__}(path='{self.path}', fs={self.fs})" + + def open( + self, + path, + *args, + **kwargs, + ): + return self.fs.open( + self._join(path), + *args, + **kwargs, + ) + + async def open_async( + self, + path, + *args, + **kwargs, + ): + return await self.fs.open_async( + self._join(path), + *args, + **kwargs, + ) diff --git a/lib/python3.12/site-packages/fsspec/implementations/ftp.py b/lib/python3.12/site-packages/fsspec/implementations/ftp.py new file mode 100644 index 0000000000000000000000000000000000000000..4186b6ce3ddff4d74151264bc6a0315b5818ce64 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/ftp.py @@ -0,0 +1,395 @@ +import os +import sys +import uuid +import warnings +from ftplib import FTP, FTP_TLS, Error, error_perm +from typing import Any + +from ..spec import AbstractBufferedFile, AbstractFileSystem +from ..utils import infer_storage_options, isfilelike + + +class FTPFileSystem(AbstractFileSystem): + """A filesystem over classic FTP""" + + root_marker = "/" + cachable = False + protocol = "ftp" + + def __init__( + self, + host, + port=21, + username=None, + password=None, + acct=None, + block_size=None, + tempdir=None, + timeout=30, + encoding="utf-8", + tls=False, + **kwargs, + ): + """ + You can use _get_kwargs_from_urls to get some kwargs from + a reasonable FTP url. + + Authentication will be anonymous if username/password are not + given. + + Parameters + ---------- + host: str + The remote server name/ip to connect to + port: int + Port to connect with + username: str or None + If authenticating, the user's identifier + password: str of None + User's password on the server, if using + acct: str or None + Some servers also need an "account" string for auth + block_size: int or None + If given, the read-ahead or write buffer size. + tempdir: str + Directory on remote to put temporary files when in a transaction + timeout: int + Timeout of the ftp connection in seconds + encoding: str + Encoding to use for directories and filenames in FTP connection + tls: bool + Use FTP-TLS, by default False + """ + super().__init__(**kwargs) + self.host = host + self.port = port + self.tempdir = tempdir or "/tmp" + self.cred = username or "", password or "", acct or "" + self.timeout = timeout + self.encoding = encoding + if block_size is not None: + self.blocksize = block_size + else: + self.blocksize = 2**16 + self.tls = tls + self._connect() + if self.tls: + self.ftp.prot_p() + + def _connect(self): + if self.tls: + ftp_cls = FTP_TLS + else: + ftp_cls = FTP + if sys.version_info >= (3, 9): + self.ftp = ftp_cls(timeout=self.timeout, encoding=self.encoding) + elif self.encoding: + warnings.warn("`encoding` not supported for python<3.9, ignoring") + self.ftp = ftp_cls(timeout=self.timeout) + else: + self.ftp = ftp_cls(timeout=self.timeout) + self.ftp.connect(self.host, self.port) + self.ftp.login(*self.cred) + + @classmethod + def _strip_protocol(cls, path): + return "/" + infer_storage_options(path)["path"].lstrip("/").rstrip("/") + + @staticmethod + def _get_kwargs_from_urls(urlpath): + out = infer_storage_options(urlpath) + out.pop("path", None) + out.pop("protocol", None) + return out + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + out = [] + if path not in self.dircache: + try: + try: + out = [ + (fn, details) + for (fn, details) in self.ftp.mlsd(path) + if fn not in [".", ".."] + and details["type"] not in ["pdir", "cdir"] + ] + except error_perm: + out = _mlsd2(self.ftp, path) # Not platform independent + for fn, details in out: + details["name"] = "/".join( + ["" if path == "/" else path, fn.lstrip("/")] + ) + if details["type"] == "file": + details["size"] = int(details["size"]) + else: + details["size"] = 0 + if details["type"] == "dir": + details["type"] = "directory" + self.dircache[path] = out + except Error: + try: + info = self.info(path) + if info["type"] == "file": + out = [(path, info)] + except (Error, IndexError) as exc: + raise FileNotFoundError(path) from exc + files = self.dircache.get(path, out) + if not detail: + return sorted([fn for fn, details in files]) + return [details for fn, details in files] + + def info(self, path, **kwargs): + # implement with direct method + path = self._strip_protocol(path) + if path == "/": + # special case, since this dir has no real entry + return {"name": "/", "size": 0, "type": "directory"} + files = self.ls(self._parent(path).lstrip("/"), True) + try: + out = next(f for f in files if f["name"] == path) + except StopIteration as exc: + raise FileNotFoundError(path) from exc + return out + + def get_file(self, rpath, lpath, **kwargs): + if self.isdir(rpath): + if not os.path.exists(lpath): + os.mkdir(lpath) + return + if isfilelike(lpath): + outfile = lpath + else: + outfile = open(lpath, "wb") + + def cb(x): + outfile.write(x) + + self.ftp.retrbinary( + f"RETR {rpath}", + blocksize=self.blocksize, + callback=cb, + ) + if not isfilelike(lpath): + outfile.close() + + def cat_file(self, path, start=None, end=None, **kwargs): + if end is not None: + return super().cat_file(path, start, end, **kwargs) + out = [] + + def cb(x): + out.append(x) + + try: + self.ftp.retrbinary( + f"RETR {path}", + blocksize=self.blocksize, + rest=start, + callback=cb, + ) + except (Error, error_perm) as orig_exc: + raise FileNotFoundError(path) from orig_exc + return b"".join(out) + + def _open( + self, + path, + mode="rb", + block_size=None, + cache_options=None, + autocommit=True, + **kwargs, + ): + path = self._strip_protocol(path) + block_size = block_size or self.blocksize + return FTPFile( + self, + path, + mode=mode, + block_size=block_size, + tempdir=self.tempdir, + autocommit=autocommit, + cache_options=cache_options, + ) + + def _rm(self, path): + path = self._strip_protocol(path) + self.ftp.delete(path) + self.invalidate_cache(self._parent(path)) + + def rm(self, path, recursive=False, maxdepth=None): + paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) + for p in reversed(paths): + if self.isfile(p): + self.rm_file(p) + else: + self.rmdir(p) + + def mkdir(self, path: str, create_parents: bool = True, **kwargs: Any) -> None: + path = self._strip_protocol(path) + parent = self._parent(path) + if parent != self.root_marker and not self.exists(parent) and create_parents: + self.mkdir(parent, create_parents=create_parents) + + self.ftp.mkd(path) + self.invalidate_cache(self._parent(path)) + + def makedirs(self, path: str, exist_ok: bool = False) -> None: + path = self._strip_protocol(path) + if self.exists(path): + # NB: "/" does not "exist" as it has no directory entry + if not exist_ok: + raise FileExistsError(f"{path} exists without `exist_ok`") + # exists_ok=True -> no-op + else: + self.mkdir(path, create_parents=True) + + def rmdir(self, path): + path = self._strip_protocol(path) + self.ftp.rmd(path) + self.invalidate_cache(self._parent(path)) + + def mv(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + self.ftp.rename(path1, path2) + self.invalidate_cache(self._parent(path1)) + self.invalidate_cache(self._parent(path2)) + + def __del__(self): + self.ftp.close() + + def invalidate_cache(self, path=None): + if path is None: + self.dircache.clear() + else: + self.dircache.pop(path, None) + super().invalidate_cache(path) + + +class TransferDone(Exception): + """Internal exception to break out of transfer""" + + pass + + +class FTPFile(AbstractBufferedFile): + """Interact with a remote FTP file with read/write buffering""" + + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + **kwargs, + ): + super().__init__( + fs, + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + if not autocommit: + self.target = self.path + self.path = "/".join([kwargs["tempdir"], str(uuid.uuid4())]) + + def commit(self): + self.fs.mv(self.path, self.target) + + def discard(self): + self.fs.rm(self.path) + + def _fetch_range(self, start, end): + """Get bytes between given byte limits + + Implemented by raising an exception in the fetch callback when the + number of bytes received reaches the requested amount. + + Will fail if the server does not respect the REST command on + retrieve requests. + """ + out = [] + total = [0] + + def callback(x): + total[0] += len(x) + if total[0] > end - start: + out.append(x[: (end - start) - total[0]]) + if end < self.size: + raise TransferDone + else: + out.append(x) + + if total[0] == end - start and end < self.size: + raise TransferDone + + try: + self.fs.ftp.retrbinary( + f"RETR {self.path}", + blocksize=self.blocksize, + rest=start, + callback=callback, + ) + except TransferDone: + try: + # stop transfer, we got enough bytes for this block + self.fs.ftp.abort() + self.fs.ftp.getmultiline() + except Error: + self.fs._connect() + + return b"".join(out) + + def _upload_chunk(self, final=False): + self.buffer.seek(0) + self.fs.ftp.storbinary( + f"STOR {self.path}", self.buffer, blocksize=self.blocksize, rest=self.offset + ) + return True + + +def _mlsd2(ftp, path="."): + """ + Fall back to using `dir` instead of `mlsd` if not supported. + + This parses a Linux style `ls -l` response to `dir`, but the response may + be platform dependent. + + Parameters + ---------- + ftp: ftplib.FTP + path: str + Expects to be given path, but defaults to ".". + """ + lines = [] + minfo = [] + ftp.dir(path, lines.append) + for line in lines: + split_line = line.split() + if len(split_line) < 9: + continue + this = ( + split_line[-1], + { + "modify": " ".join(split_line[5:8]), + "unix.owner": split_line[2], + "unix.group": split_line[3], + "unix.mode": split_line[0], + "size": split_line[4], + }, + ) + if this[1]["unix.mode"][0] == "d": + this[1]["type"] = "dir" + else: + this[1]["type"] = "file" + minfo.append(this) + return minfo diff --git a/lib/python3.12/site-packages/fsspec/implementations/git.py b/lib/python3.12/site-packages/fsspec/implementations/git.py new file mode 100644 index 0000000000000000000000000000000000000000..7b9d3539a013935e9a5652605cdd1c21cee0f0ee --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/git.py @@ -0,0 +1,115 @@ +import os + +import pygit2 + +from fsspec.spec import AbstractFileSystem + +from .memory import MemoryFile + + +class GitFileSystem(AbstractFileSystem): + """Browse the files of a local git repo at any hash/tag/branch + + (experimental backend) + """ + + root_marker = "" + cachable = True + + def __init__(self, path=None, fo=None, ref=None, **kwargs): + """ + + Parameters + ---------- + path: str (optional) + Local location of the repo (uses current directory if not given). + May be deprecated in favour of ``fo``. When used with a higher + level function such as fsspec.open(), may be of the form + "git://[path-to-repo[:]][ref@]path/to/file" (but the actual + file path should not contain "@" or ":"). + fo: str (optional) + Same as ``path``, but passed as part of a chained URL. This one + takes precedence if both are given. + ref: str (optional) + Reference to work with, could be a hash, tag or branch name. Defaults + to current working tree. Note that ``ls`` and ``open`` also take hash, + so this becomes the default for those operations + kwargs + """ + super().__init__(**kwargs) + self.repo = pygit2.Repository(fo or path or os.getcwd()) + self.ref = ref or "master" + + @classmethod + def _strip_protocol(cls, path): + path = super()._strip_protocol(path).lstrip("/") + if ":" in path: + path = path.split(":", 1)[1] + if "@" in path: + path = path.split("@", 1)[1] + return path.lstrip("/") + + def _path_to_object(self, path, ref): + comm, ref = self.repo.resolve_refish(ref or self.ref) + parts = path.split("/") + tree = comm.tree + for part in parts: + if part and isinstance(tree, pygit2.Tree): + if part not in tree: + raise FileNotFoundError(path) + tree = tree[part] + return tree + + @staticmethod + def _get_kwargs_from_urls(path): + if path.startswith("git://"): + path = path[6:] + out = {} + if ":" in path: + out["path"], path = path.split(":", 1) + if "@" in path: + out["ref"], path = path.split("@", 1) + return out + + @staticmethod + def _object_to_info(obj, path=None): + # obj.name and obj.filemode are None for the root tree! + is_dir = isinstance(obj, pygit2.Tree) + return { + "type": "directory" if is_dir else "file", + "name": ( + "/".join([path, obj.name or ""]).lstrip("/") if path else obj.name + ), + "hex": str(obj.id), + "mode": "100644" if obj.filemode is None else f"{obj.filemode:o}", + "size": 0 if is_dir else obj.size, + } + + def ls(self, path, detail=True, ref=None, **kwargs): + tree = self._path_to_object(self._strip_protocol(path), ref) + return [ + GitFileSystem._object_to_info(obj, path) + if detail + else GitFileSystem._object_to_info(obj, path)["name"] + for obj in (tree if isinstance(tree, pygit2.Tree) else [tree]) + ] + + def info(self, path, ref=None, **kwargs): + tree = self._path_to_object(self._strip_protocol(path), ref) + return GitFileSystem._object_to_info(tree, path) + + def ukey(self, path, ref=None): + return self.info(path, ref=ref)["hex"] + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + ref=None, + **kwargs, + ): + obj = self._path_to_object(path, ref or self.ref) + return MemoryFile(data=obj.data) diff --git a/lib/python3.12/site-packages/fsspec/implementations/github.py b/lib/python3.12/site-packages/fsspec/implementations/github.py new file mode 100644 index 0000000000000000000000000000000000000000..3650b8ebaa4eae3caa75a5290305fefe0a80d30b --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/github.py @@ -0,0 +1,239 @@ +import requests + +import fsspec + +from ..spec import AbstractFileSystem +from ..utils import infer_storage_options +from .memory import MemoryFile + +# TODO: add GIST backend, would be very similar + + +class GithubFileSystem(AbstractFileSystem): + """Interface to files in github + + An instance of this class provides the files residing within a remote github + repository. You may specify a point in the repos history, by SHA, branch + or tag (default is current master). + + Given that code files tend to be small, and that github does not support + retrieving partial content, we always fetch whole files. + + When using fsspec.open, allows URIs of the form: + + - "github://path/file", in which case you must specify org, repo and + may specify sha in the extra args + - 'github://org:repo@/precip/catalog.yml', where the org and repo are + part of the URI + - 'github://org:repo@sha/precip/catalog.yml', where the sha is also included + + ``sha`` can be the full or abbreviated hex of the commit you want to fetch + from, or a branch or tag name (so long as it doesn't contain special characters + like "/", "?", which would have to be HTTP-encoded). + + For authorised access, you must provide username and token, which can be made + at https://github.com/settings/tokens + """ + + url = "https://api.github.com/repos/{org}/{repo}/git/trees/{sha}" + rurl = "https://raw.githubusercontent.com/{org}/{repo}/{sha}/{path}" + protocol = "github" + timeout = (60, 60) # connect, read timeouts + + def __init__( + self, org, repo, sha=None, username=None, token=None, timeout=None, **kwargs + ): + super().__init__(**kwargs) + self.org = org + self.repo = repo + if (username is None) ^ (token is None): + raise ValueError("Auth required both username and token") + self.username = username + self.token = token + if timeout is not None: + self.timeout = timeout + if sha is None: + # look up default branch (not necessarily "master") + u = "https://api.github.com/repos/{org}/{repo}" + r = requests.get( + u.format(org=org, repo=repo), timeout=self.timeout, **self.kw + ) + r.raise_for_status() + sha = r.json()["default_branch"] + + self.root = sha + self.ls("") + + @property + def kw(self): + if self.username: + return {"auth": (self.username, self.token)} + return {} + + @classmethod + def repos(cls, org_or_user, is_org=True): + """List repo names for given org or user + + This may become the top level of the FS + + Parameters + ---------- + org_or_user: str + Name of the github org or user to query + is_org: bool (default True) + Whether the name is an organisation (True) or user (False) + + Returns + ------- + List of string + """ + r = requests.get( + f"https://api.github.com/{['users', 'orgs'][is_org]}/{org_or_user}/repos", + timeout=cls.timeout, + ) + r.raise_for_status() + return [repo["name"] for repo in r.json()] + + @property + def tags(self): + """Names of tags in the repo""" + r = requests.get( + f"https://api.github.com/repos/{self.org}/{self.repo}/tags", + timeout=self.timeout, + **self.kw, + ) + r.raise_for_status() + return [t["name"] for t in r.json()] + + @property + def branches(self): + """Names of branches in the repo""" + r = requests.get( + f"https://api.github.com/repos/{self.org}/{self.repo}/branches", + timeout=self.timeout, + **self.kw, + ) + r.raise_for_status() + return [t["name"] for t in r.json()] + + @property + def refs(self): + """Named references, tags and branches""" + return {"tags": self.tags, "branches": self.branches} + + def ls(self, path, detail=False, sha=None, _sha=None, **kwargs): + """List files at given path + + Parameters + ---------- + path: str + Location to list, relative to repo root + detail: bool + If True, returns list of dicts, one per file; if False, returns + list of full filenames only + sha: str (optional) + List at the given point in the repo history, branch or tag name or commit + SHA + _sha: str (optional) + List this specific tree object (used internally to descend into trees) + """ + path = self._strip_protocol(path) + if path == "": + _sha = sha or self.root + if _sha is None: + parts = path.rstrip("/").split("/") + so_far = "" + _sha = sha or self.root + for part in parts: + out = self.ls(so_far, True, sha=sha, _sha=_sha) + so_far += "/" + part if so_far else part + out = [o for o in out if o["name"] == so_far] + if not out: + raise FileNotFoundError(path) + out = out[0] + if out["type"] == "file": + if detail: + return [out] + else: + return path + _sha = out["sha"] + if path not in self.dircache or sha not in [self.root, None]: + r = requests.get( + self.url.format(org=self.org, repo=self.repo, sha=_sha), + timeout=self.timeout, + **self.kw, + ) + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + types = {"blob": "file", "tree": "directory"} + out = [ + { + "name": path + "/" + f["path"] if path else f["path"], + "mode": f["mode"], + "type": types[f["type"]], + "size": f.get("size", 0), + "sha": f["sha"], + } + for f in r.json()["tree"] + if f["type"] in types + ] + if sha in [self.root, None]: + self.dircache[path] = out + else: + out = self.dircache[path] + if detail: + return out + else: + return sorted([f["name"] for f in out]) + + def invalidate_cache(self, path=None): + self.dircache.clear() + + @classmethod + def _strip_protocol(cls, path): + opts = infer_storage_options(path) + if "username" not in opts: + return super()._strip_protocol(path) + return opts["path"].lstrip("/") + + @staticmethod + def _get_kwargs_from_urls(path): + opts = infer_storage_options(path) + if "username" not in opts: + return {} + out = {"org": opts["username"], "repo": opts["password"]} + if opts["host"]: + out["sha"] = opts["host"] + return out + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + sha=None, + **kwargs, + ): + if mode != "rb": + raise NotImplementedError + url = self.rurl.format( + org=self.org, repo=self.repo, path=path, sha=sha or self.root + ) + r = requests.get(url, timeout=self.timeout, **self.kw) + if r.status_code == 404: + raise FileNotFoundError(path) + r.raise_for_status() + return MemoryFile(None, None, r.content) + + def cat(self, path, recursive=False, on_error="raise", **kwargs): + paths = self.expand_path(path, recursive=recursive) + urls = [ + self.rurl.format(org=self.org, repo=self.repo, path=u, sha=self.root) + for u, sh in paths + ] + fs = fsspec.filesystem("http") + data = fs.cat(urls, on_error="return") + return {u: v for ((k, v), u) in zip(data.items(), urls)} diff --git a/lib/python3.12/site-packages/fsspec/implementations/http.py b/lib/python3.12/site-packages/fsspec/implementations/http.py new file mode 100644 index 0000000000000000000000000000000000000000..7386d4b8084846807e68b6e70e253e1ce6cb98d5 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/http.py @@ -0,0 +1,880 @@ +import asyncio +import io +import logging +import re +import weakref +from copy import copy +from urllib.parse import urlparse + +import aiohttp +import yarl + +from fsspec.asyn import AbstractAsyncStreamedFile, AsyncFileSystem, sync, sync_wrapper +from fsspec.callbacks import DEFAULT_CALLBACK +from fsspec.exceptions import FSTimeoutError +from fsspec.spec import AbstractBufferedFile +from fsspec.utils import ( + DEFAULT_BLOCK_SIZE, + glob_translate, + isfilelike, + nullcontext, + tokenize, +) + +from ..caching import AllBytes + +# https://stackoverflow.com/a/15926317/3821154 +ex = re.compile(r"""<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P[^"']+)""") +ex2 = re.compile(r"""(?Phttp[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)""") +logger = logging.getLogger("fsspec.http") + + +async def get_client(**kwargs): + return aiohttp.ClientSession(**kwargs) + + +class HTTPFileSystem(AsyncFileSystem): + """ + Simple File-System for fetching data via HTTP(S) + + ``ls()`` is implemented by loading the parent page and doing a regex + match on the result. If simple_link=True, anything of the form + "http(s)://server.com/stuff?thing=other"; otherwise only links within + HTML href tags will be used. + """ + + sep = "/" + + def __init__( + self, + simple_links=True, + block_size=None, + same_scheme=True, + size_policy=None, + cache_type="bytes", + cache_options=None, + asynchronous=False, + loop=None, + client_kwargs=None, + get_client=get_client, + encoded=False, + **storage_options, + ): + """ + NB: if this is called async, you must await set_client + + Parameters + ---------- + block_size: int + Blocks to read bytes; if 0, will default to raw requests file-like + objects instead of HTTPFile instances + simple_links: bool + If True, will consider both HTML tags and anything that looks + like a URL; if False, will consider only the former. + same_scheme: True + When doing ls/glob, if this is True, only consider paths that have + http/https matching the input URLs. + size_policy: this argument is deprecated + client_kwargs: dict + Passed to aiohttp.ClientSession, see + https://docs.aiohttp.org/en/stable/client_reference.html + For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` + get_client: Callable[..., aiohttp.ClientSession] + A callable which takes keyword arguments and constructs + an aiohttp.ClientSession. It's state will be managed by + the HTTPFileSystem class. + storage_options: key-value + Any other parameters passed on to requests + cache_type, cache_options: defaults used in open + """ + super().__init__(self, asynchronous=asynchronous, loop=loop, **storage_options) + self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE + self.simple_links = simple_links + self.same_schema = same_scheme + self.cache_type = cache_type + self.cache_options = cache_options + self.client_kwargs = client_kwargs or {} + self.get_client = get_client + self.encoded = encoded + self.kwargs = storage_options + self._session = None + + # Clean caching-related parameters from `storage_options` + # before propagating them as `request_options` through `self.kwargs`. + # TODO: Maybe rename `self.kwargs` to `self.request_options` to make + # it clearer. + request_options = copy(storage_options) + self.use_listings_cache = request_options.pop("use_listings_cache", False) + request_options.pop("listings_expiry_time", None) + request_options.pop("max_paths", None) + request_options.pop("skip_instance_cache", None) + self.kwargs = request_options + + @property + def fsid(self): + return "http" + + def encode_url(self, url): + return yarl.URL(url, encoded=self.encoded) + + @staticmethod + def close_session(loop, session): + if loop is not None and loop.is_running(): + try: + sync(loop, session.close, timeout=0.1) + return + except (TimeoutError, FSTimeoutError, NotImplementedError): + pass + connector = getattr(session, "_connector", None) + if connector is not None: + # close after loop is dead + connector._close() + + async def set_session(self): + if self._session is None: + self._session = await self.get_client(loop=self.loop, **self.client_kwargs) + if not self.asynchronous: + weakref.finalize(self, self.close_session, self.loop, self._session) + return self._session + + @classmethod + def _strip_protocol(cls, path): + """For HTTP, we always want to keep the full URL""" + return path + + @classmethod + def _parent(cls, path): + # override, since _strip_protocol is different for URLs + par = super()._parent(path) + if len(par) > 7: # "http://..." + return par + return "" + + async def _ls_real(self, url, detail=True, **kwargs): + # ignoring URL-encoded arguments + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + session = await self.set_session() + async with session.get(self.encode_url(url), **self.kwargs) as r: + self._raise_not_found_for_status(r, url) + try: + text = await r.text() + if self.simple_links: + links = ex2.findall(text) + [u[2] for u in ex.findall(text)] + else: + links = [u[2] for u in ex.findall(text)] + except UnicodeDecodeError: + links = [] # binary, not HTML + out = set() + parts = urlparse(url) + for l in links: + if isinstance(l, tuple): + l = l[1] + if l.startswith("/") and len(l) > 1: + # absolute URL on this server + l = f"{parts.scheme}://{parts.netloc}{l}" + if l.startswith("http"): + if self.same_schema and l.startswith(url.rstrip("/") + "/"): + out.add(l) + elif l.replace("https", "http").startswith( + url.replace("https", "http").rstrip("/") + "/" + ): + # allowed to cross http <-> https + out.add(l) + else: + if l not in ["..", "../"]: + # Ignore FTP-like "parent" + out.add("/".join([url.rstrip("/"), l.lstrip("/")])) + if not out and url.endswith("/"): + out = await self._ls_real(url.rstrip("/"), detail=False) + if detail: + return [ + { + "name": u, + "size": None, + "type": "directory" if u.endswith("/") else "file", + } + for u in out + ] + else: + return sorted(out) + + async def _ls(self, url, detail=True, **kwargs): + if self.use_listings_cache and url in self.dircache: + out = self.dircache[url] + else: + out = await self._ls_real(url, detail=detail, **kwargs) + self.dircache[url] = out + return out + + ls = sync_wrapper(_ls) + + def _raise_not_found_for_status(self, response, url): + """ + Raises FileNotFoundError for 404s, otherwise uses raise_for_status. + """ + if response.status == 404: + raise FileNotFoundError(url) + response.raise_for_status() + + async def _cat_file(self, url, start=None, end=None, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + + if start is not None or end is not None: + if start == end: + return b"" + headers = kw.pop("headers", {}).copy() + + headers["Range"] = await self._process_limits(url, start, end) + kw["headers"] = headers + session = await self.set_session() + async with session.get(self.encode_url(url), **kw) as r: + out = await r.read() + self._raise_not_found_for_status(r, url) + return out + + async def _get_file( + self, rpath, lpath, chunk_size=5 * 2**20, callback=DEFAULT_CALLBACK, **kwargs + ): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(rpath) + session = await self.set_session() + async with session.get(self.encode_url(rpath), **kw) as r: + try: + size = int(r.headers["content-length"]) + except (ValueError, KeyError): + size = None + + callback.set_size(size) + self._raise_not_found_for_status(r, rpath) + if isfilelike(lpath): + outfile = lpath + else: + outfile = open(lpath, "wb") # noqa: ASYNC101, ASYNC230 + + try: + chunk = True + while chunk: + chunk = await r.content.read(chunk_size) + outfile.write(chunk) + callback.relative_update(len(chunk)) + finally: + if not isfilelike(lpath): + outfile.close() + + async def _put_file( + self, + lpath, + rpath, + chunk_size=5 * 2**20, + callback=DEFAULT_CALLBACK, + method="post", + mode="overwrite", + **kwargs, + ): + if mode != "overwrite": + raise NotImplementedError("Exclusive write") + + async def gen_chunks(): + # Support passing arbitrary file-like objects + # and use them instead of streams. + if isinstance(lpath, io.IOBase): + context = nullcontext(lpath) + use_seek = False # might not support seeking + else: + context = open(lpath, "rb") # noqa: ASYNC101, ASYNC230 + use_seek = True + + with context as f: + if use_seek: + callback.set_size(f.seek(0, 2)) + f.seek(0) + else: + callback.set_size(getattr(f, "size", None)) + + chunk = f.read(chunk_size) + while chunk: + yield chunk + callback.relative_update(len(chunk)) + chunk = f.read(chunk_size) + + kw = self.kwargs.copy() + kw.update(kwargs) + session = await self.set_session() + + method = method.lower() + if method not in ("post", "put"): + raise ValueError( + f"method has to be either 'post' or 'put', not: {method!r}" + ) + + meth = getattr(session, method) + async with meth(self.encode_url(rpath), data=gen_chunks(), **kw) as resp: + self._raise_not_found_for_status(resp, rpath) + + async def _exists(self, path, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + try: + logger.debug(path) + session = await self.set_session() + r = await session.get(self.encode_url(path), **kw) + async with r: + return r.status < 400 + except aiohttp.ClientError: + return False + + async def _isfile(self, path, **kwargs): + return await self._exists(path, **kwargs) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=None, # XXX: This differs from the base class. + cache_type=None, + cache_options=None, + size=None, + **kwargs, + ): + """Make a file-like object + + Parameters + ---------- + path: str + Full URL with protocol + mode: string + must be "rb" + block_size: int or None + Bytes to download in one request; use instance value if None. If + zero, will return a streaming Requests file-like instance. + kwargs: key-value + Any other parameters, passed to requests calls + """ + if mode != "rb": + raise NotImplementedError + block_size = block_size if block_size is not None else self.block_size + kw = self.kwargs.copy() + kw["asynchronous"] = self.asynchronous + kw.update(kwargs) + info = {} + size = size or info.update(self.info(path, **kwargs)) or info["size"] + session = sync(self.loop, self.set_session) + if block_size and size and info.get("partial", True): + return HTTPFile( + self, + path, + session=session, + block_size=block_size, + mode=mode, + size=size, + cache_type=cache_type or self.cache_type, + cache_options=cache_options or self.cache_options, + loop=self.loop, + **kw, + ) + else: + return HTTPStreamFile( + self, + path, + mode=mode, + loop=self.loop, + session=session, + **kw, + ) + + async def open_async(self, path, mode="rb", size=None, **kwargs): + session = await self.set_session() + if size is None: + try: + size = (await self._info(path, **kwargs))["size"] + except FileNotFoundError: + pass + return AsyncStreamFile( + self, + path, + loop=self.loop, + session=session, + size=size, + **kwargs, + ) + + def ukey(self, url): + """Unique identifier; assume HTTP files are static, unchanging""" + return tokenize(url, self.kwargs, self.protocol) + + async def _info(self, url, **kwargs): + """Get info of URL + + Tries to access location via HEAD, and then GET methods, but does + not fetch the data. + + It is possible that the server does not supply any size information, in + which case size will be given as None (and certain operations on the + corresponding file will not work). + """ + info = {} + session = await self.set_session() + + for policy in ["head", "get"]: + try: + info.update( + await _file_info( + self.encode_url(url), + size_policy=policy, + session=session, + **self.kwargs, + **kwargs, + ) + ) + if info.get("size") is not None: + break + except Exception as exc: + if policy == "get": + # If get failed, then raise a FileNotFoundError + raise FileNotFoundError(url) from exc + logger.debug("", exc_info=exc) + + return {"name": url, "size": None, **info, "type": "file"} + + async def _glob(self, path, maxdepth=None, **kwargs): + """ + Find files by glob-matching. + + This implementation is idntical to the one in AbstractFileSystem, + but "?" is not considered as a character for globbing, because it is + so common in URLs, often identifying the "query" part. + """ + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + import re + + ends_with_slash = path.endswith("/") # _strip_protocol strips trailing slash + path = self._strip_protocol(path) + append_slash_to_dirname = ends_with_slash or path.endswith(("/**", "/*")) + idx_star = path.find("*") if path.find("*") >= 0 else len(path) + idx_brace = path.find("[") if path.find("[") >= 0 else len(path) + + min_idx = min(idx_star, idx_brace) + + detail = kwargs.pop("detail", False) + + if not has_magic(path): + if await self._exists(path, **kwargs): + if not detail: + return [path] + else: + return {path: await self._info(path, **kwargs)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:min_idx]: + min_idx = path[:min_idx].rindex("/") + root = path[: min_idx + 1] + depth = path[min_idx + 1 :].count("/") + 1 + else: + root = "" + depth = path[min_idx + 1 :].count("/") + 1 + + if "**" in path: + if maxdepth is not None: + idx_double_stars = path.find("**") + depth_double_stars = path[idx_double_stars:].count("/") + 1 + depth = depth - depth_double_stars + maxdepth + else: + depth = None + + allpaths = await self._find( + root, maxdepth=depth, withdirs=True, detail=True, **kwargs + ) + + pattern = glob_translate(path + ("/" if ends_with_slash else "")) + pattern = re.compile(pattern) + + out = { + ( + p.rstrip("/") + if not append_slash_to_dirname + and info["type"] == "directory" + and p.endswith("/") + else p + ): info + for p, info in sorted(allpaths.items()) + if pattern.match(p.rstrip("/")) + } + + if detail: + return out + else: + return list(out) + + async def _isdir(self, path): + # override, since all URLs are (also) files + try: + return bool(await self._ls(path)) + except (FileNotFoundError, ValueError): + return False + + async def _pipe_file(self, path, value, mode="overwrite", **kwargs): + """ + Write bytes to a remote file over HTTP. + + Parameters + ---------- + path : str + Target URL where the data should be written + value : bytes + Data to be written + mode : str + How to write to the file - 'overwrite' or 'append' + **kwargs : dict + Additional parameters to pass to the HTTP request + """ + url = self._strip_protocol(path) + headers = kwargs.pop("headers", {}) + headers["Content-Length"] = str(len(value)) + + session = await self.set_session() + + async with session.put(url, data=value, headers=headers, **kwargs) as r: + r.raise_for_status() + + +class HTTPFile(AbstractBufferedFile): + """ + A file-like object pointing to a remote HTTP(S) resource + + Supports only reading, with read-ahead of a predetermined block-size. + + In the case that the server does not supply the filesize, only reading of + the complete file in one go is supported. + + Parameters + ---------- + url: str + Full URL of the remote resource, including the protocol + session: aiohttp.ClientSession or None + All calls will be made within this session, to avoid restarting + connections where the server allows this + block_size: int or None + The amount of read-ahead to do, in bytes. Default is 5MB, or the value + configured for the FileSystem creating this file + size: None or int + If given, this is the size of the file in bytes, and we don't attempt + to call the server to find the value. + kwargs: all other key-values are passed to requests calls. + """ + + def __init__( + self, + fs, + url, + session=None, + block_size=None, + mode="rb", + cache_type="bytes", + cache_options=None, + size=None, + loop=None, + asynchronous=False, + **kwargs, + ): + if mode != "rb": + raise NotImplementedError("File mode not supported") + self.asynchronous = asynchronous + self.loop = loop + self.url = url + self.session = session + self.details = {"name": url, "size": size, "type": "file"} + super().__init__( + fs=fs, + path=url, + mode=mode, + block_size=block_size, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + + def read(self, length=-1): + """Read bytes from file + + Parameters + ---------- + length: int + Read up to this many bytes. If negative, read all content to end of + file. If the server has not supplied the filesize, attempting to + read only part of the data will raise a ValueError. + """ + if ( + (length < 0 and self.loc == 0) # explicit read all + # but not when the size is known and fits into a block anyways + and not (self.size is not None and self.size <= self.blocksize) + ): + self._fetch_all() + if self.size is None: + if length < 0: + self._fetch_all() + else: + length = min(self.size - self.loc, length) + return super().read(length) + + async def async_fetch_all(self): + """Read whole file in one shot, without caching + + This is only called when position is still at zero, + and read() is called without a byte-count. + """ + logger.debug(f"Fetch all for {self}") + if not isinstance(self.cache, AllBytes): + r = await self.session.get(self.fs.encode_url(self.url), **self.kwargs) + async with r: + r.raise_for_status() + out = await r.read() + self.cache = AllBytes( + size=len(out), fetcher=None, blocksize=None, data=out + ) + self.size = len(out) + + _fetch_all = sync_wrapper(async_fetch_all) + + def _parse_content_range(self, headers): + """Parse the Content-Range header""" + s = headers.get("Content-Range", "") + m = re.match(r"bytes (\d+-\d+|\*)/(\d+|\*)", s) + if not m: + return None, None, None + + if m[1] == "*": + start = end = None + else: + start, end = [int(x) for x in m[1].split("-")] + total = None if m[2] == "*" else int(m[2]) + return start, end, total + + async def async_fetch_range(self, start, end): + """Download a block of data + + The expectation is that the server returns only the requested bytes, + with HTTP code 206. If this is not the case, we first check the headers, + and then stream the output - if the data size is bigger than we + requested, an exception is raised. + """ + logger.debug(f"Fetch range for {self}: {start}-{end}") + kwargs = self.kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + logger.debug(f"{self.url} : {headers['Range']}") + r = await self.session.get( + self.fs.encode_url(self.url), headers=headers, **kwargs + ) + async with r: + if r.status == 416: + # range request outside file + return b"" + r.raise_for_status() + + # If the server has handled the range request, it should reply + # with status 206 (partial content). But we'll guess that a suitable + # Content-Range header or a Content-Length no more than the + # requested range also mean we have got the desired range. + response_is_range = ( + r.status == 206 + or self._parse_content_range(r.headers)[0] == start + or int(r.headers.get("Content-Length", end + 1)) <= end - start + ) + + if response_is_range: + # partial content, as expected + out = await r.read() + elif start > 0: + raise ValueError( + "The HTTP server doesn't appear to support range requests. " + "Only reading this file from the beginning is supported. " + "Open with block_size=0 for a streaming file interface." + ) + else: + # Response is not a range, but we want the start of the file, + # so we can read the required amount anyway. + cl = 0 + out = [] + while True: + chunk = await r.content.read(2**20) + # data size unknown, let's read until we have enough + if chunk: + out.append(chunk) + cl += len(chunk) + if cl > end - start: + break + else: + break + out = b"".join(out)[: end - start] + return out + + _fetch_range = sync_wrapper(async_fetch_range) + + +magic_check = re.compile("([*[])") + + +def has_magic(s): + match = magic_check.search(s) + return match is not None + + +class HTTPStreamFile(AbstractBufferedFile): + def __init__(self, fs, url, mode="rb", loop=None, session=None, **kwargs): + self.asynchronous = kwargs.pop("asynchronous", False) + self.url = url + self.loop = loop + self.session = session + if mode != "rb": + raise ValueError + self.details = {"name": url, "size": None} + super().__init__(fs=fs, path=url, mode=mode, cache_type="none", **kwargs) + + async def cor(): + r = await self.session.get(self.fs.encode_url(url), **kwargs).__aenter__() + self.fs._raise_not_found_for_status(r, url) + return r + + self.r = sync(self.loop, cor) + self.loop = fs.loop + + def seek(self, loc, whence=0): + if loc == 0 and whence == 1: + return + if loc == self.loc and whence == 0: + return + raise ValueError("Cannot seek streaming HTTP file") + + async def _read(self, num=-1): + out = await self.r.content.read(num) + self.loc += len(out) + return out + + read = sync_wrapper(_read) + + async def _close(self): + self.r.close() + + def close(self): + asyncio.run_coroutine_threadsafe(self._close(), self.loop) + super().close() + + +class AsyncStreamFile(AbstractAsyncStreamedFile): + def __init__( + self, fs, url, mode="rb", loop=None, session=None, size=None, **kwargs + ): + self.url = url + self.session = session + self.r = None + if mode != "rb": + raise ValueError + self.details = {"name": url, "size": None} + self.kwargs = kwargs + super().__init__(fs=fs, path=url, mode=mode, cache_type="none") + self.size = size + + async def read(self, num=-1): + if self.r is None: + r = await self.session.get( + self.fs.encode_url(self.url), **self.kwargs + ).__aenter__() + self.fs._raise_not_found_for_status(r, self.url) + self.r = r + out = await self.r.content.read(num) + self.loc += len(out) + return out + + async def close(self): + if self.r is not None: + self.r.close() + self.r = None + await super().close() + + +async def get_range(session, url, start, end, file=None, **kwargs): + # explicit get a range when we know it must be safe + kwargs = kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + r = await session.get(url, headers=headers, **kwargs) + r.raise_for_status() + async with r: + out = await r.read() + if file: + with open(file, "r+b") as f: # noqa: ASYNC101, ASYNC230 + f.seek(start) + f.write(out) + else: + return out + + +async def _file_info(url, session, size_policy="head", **kwargs): + """Call HEAD on the server to get details about the file (size/checksum etc.) + + Default operation is to explicitly allow redirects and use encoding + 'identity' (no compression) to get the true size of the target. + """ + logger.debug("Retrieve file size for %s", url) + kwargs = kwargs.copy() + ar = kwargs.pop("allow_redirects", True) + head = kwargs.get("headers", {}).copy() + head["Accept-Encoding"] = "identity" + kwargs["headers"] = head + + info = {} + if size_policy == "head": + r = await session.head(url, allow_redirects=ar, **kwargs) + elif size_policy == "get": + r = await session.get(url, allow_redirects=ar, **kwargs) + else: + raise TypeError(f'size_policy must be "head" or "get", got {size_policy}') + async with r: + r.raise_for_status() + + if "Content-Length" in r.headers: + # Some servers may choose to ignore Accept-Encoding and return + # compressed content, in which case the returned size is unreliable. + if "Content-Encoding" not in r.headers or r.headers["Content-Encoding"] in [ + "identity", + "", + ]: + info["size"] = int(r.headers["Content-Length"]) + elif "Content-Range" in r.headers: + info["size"] = int(r.headers["Content-Range"].split("/")[1]) + + if "Content-Type" in r.headers: + info["mimetype"] = r.headers["Content-Type"].partition(";")[0] + + if r.headers.get("Accept-Ranges") == "none": + # Some servers may explicitly discourage partial content requests, but + # the lack of "Accept-Ranges" does not always indicate they would fail + info["partial"] = False + + info["url"] = str(r.url) + + for checksum_field in ["ETag", "Content-MD5", "Digest"]: + if r.headers.get(checksum_field): + info[checksum_field] = r.headers[checksum_field] + + return info + + +async def _file_size(url, session=None, *args, **kwargs): + if session is None: + session = await get_client() + info = await _file_info(url, session=session, *args, **kwargs) + return info.get("size") + + +file_size = sync_wrapper(_file_size) diff --git a/lib/python3.12/site-packages/fsspec/implementations/http_sync.py b/lib/python3.12/site-packages/fsspec/implementations/http_sync.py new file mode 100644 index 0000000000000000000000000000000000000000..2a1caf1b0aaa0c8b2bc58f5364aac5ee5cf44d1a --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/http_sync.py @@ -0,0 +1,932 @@ +"""This file is largely copied from http.py""" + +import io +import logging +import re +import urllib.error +import urllib.parse +from copy import copy +from json import dumps, loads +from urllib.parse import urlparse + +try: + import yarl +except (ImportError, ModuleNotFoundError, OSError): + yarl = False + +from fsspec.callbacks import _DEFAULT_CALLBACK +from fsspec.registry import register_implementation +from fsspec.spec import AbstractBufferedFile, AbstractFileSystem +from fsspec.utils import DEFAULT_BLOCK_SIZE, isfilelike, nullcontext, tokenize + +from ..caching import AllBytes + +# https://stackoverflow.com/a/15926317/3821154 +ex = re.compile(r"""<(a|A)\s+(?:[^>]*?\s+)?(href|HREF)=["'](?P[^"']+)""") +ex2 = re.compile(r"""(?Phttp[s]?://[-a-zA-Z0-9@:%_+.~#?&/=]+)""") +logger = logging.getLogger("fsspec.http") + + +class JsHttpException(urllib.error.HTTPError): ... + + +class StreamIO(io.BytesIO): + # fake class, so you can set attributes on it + # will eventually actually stream + ... + + +class ResponseProxy: + """Looks like a requests response""" + + def __init__(self, req, stream=False): + self.request = req + self.stream = stream + self._data = None + self._headers = None + + @property + def raw(self): + if self._data is None: + b = self.request.response.to_bytes() + if self.stream: + self._data = StreamIO(b) + else: + self._data = b + return self._data + + def close(self): + if hasattr(self, "_data"): + del self._data + + @property + def headers(self): + if self._headers is None: + self._headers = dict( + [ + _.split(": ") + for _ in self.request.getAllResponseHeaders().strip().split("\r\n") + ] + ) + return self._headers + + @property + def status_code(self): + return int(self.request.status) + + def raise_for_status(self): + if not self.ok: + raise JsHttpException( + self.url, self.status_code, self.reason, self.headers, None + ) + + def iter_content(self, chunksize, *_, **__): + while True: + out = self.raw.read(chunksize) + if out: + yield out + else: + break + + @property + def reason(self): + return self.request.statusText + + @property + def ok(self): + return self.status_code < 400 + + @property + def url(self): + return self.request.response.responseURL + + @property + def text(self): + # TODO: encoding from headers + return self.content.decode() + + @property + def content(self): + self.stream = False + return self.raw + + @property + def json(self): + return loads(self.text) + + +class RequestsSessionShim: + def __init__(self): + self.headers = {} + + def request( + self, + method, + url, + params=None, + data=None, + headers=None, + cookies=None, + files=None, + auth=None, + timeout=None, + allow_redirects=None, + proxies=None, + hooks=None, + stream=None, + verify=None, + cert=None, + json=None, + ): + from js import Blob, XMLHttpRequest + + logger.debug("JS request: %s %s", method, url) + + if cert or verify or proxies or files or cookies or hooks: + raise NotImplementedError + if data and json: + raise ValueError("Use json= or data=, not both") + req = XMLHttpRequest.new() + extra = auth if auth else () + if params: + url = f"{url}?{urllib.parse.urlencode(params)}" + req.open(method, url, False, *extra) + if timeout: + req.timeout = timeout + if headers: + for k, v in headers.items(): + req.setRequestHeader(k, v) + + req.setRequestHeader("Accept", "application/octet-stream") + req.responseType = "arraybuffer" + if json: + blob = Blob.new([dumps(data)], {type: "application/json"}) + req.send(blob) + elif data: + if isinstance(data, io.IOBase): + data = data.read() + blob = Blob.new([data], {type: "application/octet-stream"}) + req.send(blob) + else: + req.send(None) + return ResponseProxy(req, stream=stream) + + def get(self, url, **kwargs): + return self.request("GET", url, **kwargs) + + def head(self, url, **kwargs): + return self.request("HEAD", url, **kwargs) + + def post(self, url, **kwargs): + return self.request("POST}", url, **kwargs) + + def put(self, url, **kwargs): + return self.request("PUT", url, **kwargs) + + def patch(self, url, **kwargs): + return self.request("PATCH", url, **kwargs) + + def delete(self, url, **kwargs): + return self.request("DELETE", url, **kwargs) + + +class HTTPFileSystem(AbstractFileSystem): + """ + Simple File-System for fetching data via HTTP(S) + + This is the BLOCKING version of the normal HTTPFileSystem. It uses + requests in normal python and the JS runtime in pyodide. + + ***This implementation is extremely experimental, do not use unless + you are testing pyodide/pyscript integration*** + """ + + protocol = ("http", "https", "sync_http", "sync_https") + sep = "/" + + def __init__( + self, + simple_links=True, + block_size=None, + same_scheme=True, + cache_type="readahead", + cache_options=None, + client_kwargs=None, + encoded=False, + **storage_options, + ): + """ + + Parameters + ---------- + block_size: int + Blocks to read bytes; if 0, will default to raw requests file-like + objects instead of HTTPFile instances + simple_links: bool + If True, will consider both HTML tags and anything that looks + like a URL; if False, will consider only the former. + same_scheme: True + When doing ls/glob, if this is True, only consider paths that have + http/https matching the input URLs. + size_policy: this argument is deprecated + client_kwargs: dict + Passed to aiohttp.ClientSession, see + https://docs.aiohttp.org/en/stable/client_reference.html + For example, ``{'auth': aiohttp.BasicAuth('user', 'pass')}`` + storage_options: key-value + Any other parameters passed on to requests + cache_type, cache_options: defaults used in open + """ + super().__init__(self, **storage_options) + self.block_size = block_size if block_size is not None else DEFAULT_BLOCK_SIZE + self.simple_links = simple_links + self.same_schema = same_scheme + self.cache_type = cache_type + self.cache_options = cache_options + self.client_kwargs = client_kwargs or {} + self.encoded = encoded + self.kwargs = storage_options + + try: + import js # noqa: F401 + + logger.debug("Starting JS session") + self.session = RequestsSessionShim() + self.js = True + except Exception as e: + import requests + + logger.debug("Starting cpython session because of: %s", e) + self.session = requests.Session(**(client_kwargs or {})) + self.js = False + + request_options = copy(storage_options) + self.use_listings_cache = request_options.pop("use_listings_cache", False) + request_options.pop("listings_expiry_time", None) + request_options.pop("max_paths", None) + request_options.pop("skip_instance_cache", None) + self.kwargs = request_options + + @property + def fsid(self): + return "http_sync" + + def encode_url(self, url): + if yarl: + return yarl.URL(url, encoded=self.encoded) + return url + + @classmethod + def _strip_protocol(cls, path: str) -> str: + """For HTTP, we always want to keep the full URL""" + path = path.replace("http_sync://", "http://").replace( + "https_sync://", "https://" + ) + return path + + @classmethod + def _parent(cls, path): + # override, since _strip_protocol is different for URLs + par = super()._parent(path) + if len(par) > 7: # "http://..." + return par + return "" + + def _ls_real(self, url, detail=True, **kwargs): + # ignoring URL-encoded arguments + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + r = self.session.get(self.encode_url(url), **self.kwargs) + self._raise_not_found_for_status(r, url) + text = r.text + if self.simple_links: + links = ex2.findall(text) + [u[2] for u in ex.findall(text)] + else: + links = [u[2] for u in ex.findall(text)] + out = set() + parts = urlparse(url) + for l in links: + if isinstance(l, tuple): + l = l[1] + if l.startswith("/") and len(l) > 1: + # absolute URL on this server + l = parts.scheme + "://" + parts.netloc + l + if l.startswith("http"): + if self.same_schema and l.startswith(url.rstrip("/") + "/"): + out.add(l) + elif l.replace("https", "http").startswith( + url.replace("https", "http").rstrip("/") + "/" + ): + # allowed to cross http <-> https + out.add(l) + else: + if l not in ["..", "../"]: + # Ignore FTP-like "parent" + out.add("/".join([url.rstrip("/"), l.lstrip("/")])) + if not out and url.endswith("/"): + out = self._ls_real(url.rstrip("/"), detail=False) + if detail: + return [ + { + "name": u, + "size": None, + "type": "directory" if u.endswith("/") else "file", + } + for u in out + ] + else: + return sorted(out) + + def ls(self, url, detail=True, **kwargs): + if self.use_listings_cache and url in self.dircache: + out = self.dircache[url] + else: + out = self._ls_real(url, detail=detail, **kwargs) + self.dircache[url] = out + return out + + def _raise_not_found_for_status(self, response, url): + """ + Raises FileNotFoundError for 404s, otherwise uses raise_for_status. + """ + if response.status_code == 404: + raise FileNotFoundError(url) + response.raise_for_status() + + def cat_file(self, url, start=None, end=None, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(url) + + if start is not None or end is not None: + if start == end: + return b"" + headers = kw.pop("headers", {}).copy() + + headers["Range"] = self._process_limits(url, start, end) + kw["headers"] = headers + r = self.session.get(self.encode_url(url), **kw) + self._raise_not_found_for_status(r, url) + return r.content + + def get_file( + self, rpath, lpath, chunk_size=5 * 2**20, callback=_DEFAULT_CALLBACK, **kwargs + ): + kw = self.kwargs.copy() + kw.update(kwargs) + logger.debug(rpath) + r = self.session.get(self.encode_url(rpath), **kw) + try: + size = int( + r.headers.get("content-length", None) + or r.headers.get("Content-Length", None) + ) + except (ValueError, KeyError, TypeError): + size = None + + callback.set_size(size) + self._raise_not_found_for_status(r, rpath) + if not isfilelike(lpath): + lpath = open(lpath, "wb") + for chunk in r.iter_content(chunk_size, decode_unicode=False): + lpath.write(chunk) + callback.relative_update(len(chunk)) + + def put_file( + self, + lpath, + rpath, + chunk_size=5 * 2**20, + callback=_DEFAULT_CALLBACK, + method="post", + **kwargs, + ): + def gen_chunks(): + # Support passing arbitrary file-like objects + # and use them instead of streams. + if isinstance(lpath, io.IOBase): + context = nullcontext(lpath) + use_seek = False # might not support seeking + else: + context = open(lpath, "rb") + use_seek = True + + with context as f: + if use_seek: + callback.set_size(f.seek(0, 2)) + f.seek(0) + else: + callback.set_size(getattr(f, "size", None)) + + chunk = f.read(chunk_size) + while chunk: + yield chunk + callback.relative_update(len(chunk)) + chunk = f.read(chunk_size) + + kw = self.kwargs.copy() + kw.update(kwargs) + + method = method.lower() + if method not in ("post", "put"): + raise ValueError( + f"method has to be either 'post' or 'put', not: {method!r}" + ) + + meth = getattr(self.session, method) + resp = meth(rpath, data=gen_chunks(), **kw) + self._raise_not_found_for_status(resp, rpath) + + def _process_limits(self, url, start, end): + """Helper for "Range"-based _cat_file""" + size = None + suff = False + if start is not None and start < 0: + # if start is negative and end None, end is the "suffix length" + if end is None: + end = -start + start = "" + suff = True + else: + size = size or self.info(url)["size"] + start = size + start + elif start is None: + start = 0 + if not suff: + if end is not None and end < 0: + if start is not None: + size = size or self.info(url)["size"] + end = size + end + elif end is None: + end = "" + if isinstance(end, int): + end -= 1 # bytes range is inclusive + return f"bytes={start}-{end}" + + def exists(self, path, **kwargs): + kw = self.kwargs.copy() + kw.update(kwargs) + try: + logger.debug(path) + r = self.session.get(self.encode_url(path), **kw) + return r.status_code < 400 + except Exception: + return False + + def isfile(self, path, **kwargs): + return self.exists(path, **kwargs) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=None, # XXX: This differs from the base class. + cache_type=None, + cache_options=None, + size=None, + **kwargs, + ): + """Make a file-like object + + Parameters + ---------- + path: str + Full URL with protocol + mode: string + must be "rb" + block_size: int or None + Bytes to download in one request; use instance value if None. If + zero, will return a streaming Requests file-like instance. + kwargs: key-value + Any other parameters, passed to requests calls + """ + if mode != "rb": + raise NotImplementedError + block_size = block_size if block_size is not None else self.block_size + kw = self.kwargs.copy() + kw.update(kwargs) + size = size or self.info(path, **kwargs)["size"] + if block_size and size: + return HTTPFile( + self, + path, + session=self.session, + block_size=block_size, + mode=mode, + size=size, + cache_type=cache_type or self.cache_type, + cache_options=cache_options or self.cache_options, + **kw, + ) + else: + return HTTPStreamFile( + self, + path, + mode=mode, + session=self.session, + **kw, + ) + + def ukey(self, url): + """Unique identifier; assume HTTP files are static, unchanging""" + return tokenize(url, self.kwargs, self.protocol) + + def info(self, url, **kwargs): + """Get info of URL + + Tries to access location via HEAD, and then GET methods, but does + not fetch the data. + + It is possible that the server does not supply any size information, in + which case size will be given as None (and certain operations on the + corresponding file will not work). + """ + info = {} + for policy in ["head", "get"]: + try: + info.update( + _file_info( + self.encode_url(url), + size_policy=policy, + session=self.session, + **self.kwargs, + **kwargs, + ) + ) + if info.get("size") is not None: + break + except Exception as exc: + if policy == "get": + # If get failed, then raise a FileNotFoundError + raise FileNotFoundError(url) from exc + logger.debug(str(exc)) + + return {"name": url, "size": None, **info, "type": "file"} + + def glob(self, path, maxdepth=None, **kwargs): + """ + Find files by glob-matching. + + This implementation is idntical to the one in AbstractFileSystem, + but "?" is not considered as a character for globbing, because it is + so common in URLs, often identifying the "query" part. + """ + import re + + ends = path.endswith("/") + path = self._strip_protocol(path) + indstar = path.find("*") if path.find("*") >= 0 else len(path) + indbrace = path.find("[") if path.find("[") >= 0 else len(path) + + ind = min(indstar, indbrace) + + detail = kwargs.pop("detail", False) + + if not has_magic(path): + root = path + depth = 1 + if ends: + path += "/*" + elif self.exists(path): + if not detail: + return [path] + else: + return {path: self.info(path)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:ind]: + ind2 = path[:ind].rindex("/") + root = path[: ind2 + 1] + depth = None if "**" in path else path[ind2 + 1 :].count("/") + 1 + else: + root = "" + depth = None if "**" in path else path[ind + 1 :].count("/") + 1 + + allpaths = self.find( + root, maxdepth=maxdepth or depth, withdirs=True, detail=True, **kwargs + ) + # Escape characters special to python regex, leaving our supported + # special characters in place. + # See https://www.gnu.org/software/bash/manual/html_node/Pattern-Matching.html + # for shell globbing details. + pattern = ( + "^" + + ( + path.replace("\\", r"\\") + .replace(".", r"\.") + .replace("+", r"\+") + .replace("//", "/") + .replace("(", r"\(") + .replace(")", r"\)") + .replace("|", r"\|") + .replace("^", r"\^") + .replace("$", r"\$") + .replace("{", r"\{") + .replace("}", r"\}") + .rstrip("/") + ) + + "$" + ) + pattern = re.sub("[*]{2}", "=PLACEHOLDER=", pattern) + pattern = re.sub("[*]", "[^/]*", pattern) + pattern = re.compile(pattern.replace("=PLACEHOLDER=", ".*")) + out = { + p: allpaths[p] + for p in sorted(allpaths) + if pattern.match(p.replace("//", "/").rstrip("/")) + } + if detail: + return out + else: + return list(out) + + def isdir(self, path): + # override, since all URLs are (also) files + try: + return bool(self.ls(path)) + except (FileNotFoundError, ValueError): + return False + + +class HTTPFile(AbstractBufferedFile): + """ + A file-like object pointing to a remove HTTP(S) resource + + Supports only reading, with read-ahead of a predermined block-size. + + In the case that the server does not supply the filesize, only reading of + the complete file in one go is supported. + + Parameters + ---------- + url: str + Full URL of the remote resource, including the protocol + session: requests.Session or None + All calls will be made within this session, to avoid restarting + connections where the server allows this + block_size: int or None + The amount of read-ahead to do, in bytes. Default is 5MB, or the value + configured for the FileSystem creating this file + size: None or int + If given, this is the size of the file in bytes, and we don't attempt + to call the server to find the value. + kwargs: all other key-values are passed to requests calls. + """ + + def __init__( + self, + fs, + url, + session=None, + block_size=None, + mode="rb", + cache_type="bytes", + cache_options=None, + size=None, + **kwargs, + ): + if mode != "rb": + raise NotImplementedError("File mode not supported") + self.url = url + self.session = session + self.details = {"name": url, "size": size, "type": "file"} + super().__init__( + fs=fs, + path=url, + mode=mode, + block_size=block_size, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + + def read(self, length=-1): + """Read bytes from file + + Parameters + ---------- + length: int + Read up to this many bytes. If negative, read all content to end of + file. If the server has not supplied the filesize, attempting to + read only part of the data will raise a ValueError. + """ + if ( + (length < 0 and self.loc == 0) # explicit read all + # but not when the size is known and fits into a block anyways + and not (self.size is not None and self.size <= self.blocksize) + ): + self._fetch_all() + if self.size is None: + if length < 0: + self._fetch_all() + else: + length = min(self.size - self.loc, length) + return super().read(length) + + def _fetch_all(self): + """Read whole file in one shot, without caching + + This is only called when position is still at zero, + and read() is called without a byte-count. + """ + logger.debug(f"Fetch all for {self}") + if not isinstance(self.cache, AllBytes): + r = self.session.get(self.fs.encode_url(self.url), **self.kwargs) + r.raise_for_status() + out = r.content + self.cache = AllBytes(size=len(out), fetcher=None, blocksize=None, data=out) + self.size = len(out) + + def _parse_content_range(self, headers): + """Parse the Content-Range header""" + s = headers.get("Content-Range", "") + m = re.match(r"bytes (\d+-\d+|\*)/(\d+|\*)", s) + if not m: + return None, None, None + + if m[1] == "*": + start = end = None + else: + start, end = [int(x) for x in m[1].split("-")] + total = None if m[2] == "*" else int(m[2]) + return start, end, total + + def _fetch_range(self, start, end): + """Download a block of data + + The expectation is that the server returns only the requested bytes, + with HTTP code 206. If this is not the case, we first check the headers, + and then stream the output - if the data size is bigger than we + requested, an exception is raised. + """ + logger.debug(f"Fetch range for {self}: {start}-{end}") + kwargs = self.kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + logger.debug("%s : %s", self.url, headers["Range"]) + r = self.session.get(self.fs.encode_url(self.url), headers=headers, **kwargs) + if r.status_code == 416: + # range request outside file + return b"" + r.raise_for_status() + + # If the server has handled the range request, it should reply + # with status 206 (partial content). But we'll guess that a suitable + # Content-Range header or a Content-Length no more than the + # requested range also mean we have got the desired range. + cl = r.headers.get("Content-Length", r.headers.get("content-length", end + 1)) + response_is_range = ( + r.status_code == 206 + or self._parse_content_range(r.headers)[0] == start + or int(cl) <= end - start + ) + + if response_is_range: + # partial content, as expected + out = r.content + elif start > 0: + raise ValueError( + "The HTTP server doesn't appear to support range requests. " + "Only reading this file from the beginning is supported. " + "Open with block_size=0 for a streaming file interface." + ) + else: + # Response is not a range, but we want the start of the file, + # so we can read the required amount anyway. + cl = 0 + out = [] + for chunk in r.iter_content(2**20, False): + out.append(chunk) + cl += len(chunk) + out = b"".join(out)[: end - start] + return out + + +magic_check = re.compile("([*[])") + + +def has_magic(s): + match = magic_check.search(s) + return match is not None + + +class HTTPStreamFile(AbstractBufferedFile): + def __init__(self, fs, url, mode="rb", session=None, **kwargs): + self.url = url + self.session = session + if mode != "rb": + raise ValueError + self.details = {"name": url, "size": None} + super().__init__(fs=fs, path=url, mode=mode, cache_type="readahead", **kwargs) + + r = self.session.get(self.fs.encode_url(url), stream=True, **kwargs) + self.fs._raise_not_found_for_status(r, url) + self.it = r.iter_content(1024, False) + self.leftover = b"" + + self.r = r + + def seek(self, *args, **kwargs): + raise ValueError("Cannot seek streaming HTTP file") + + def read(self, num=-1): + bufs = [self.leftover] + leng = len(self.leftover) + while leng < num or num < 0: + try: + out = self.it.__next__() + except StopIteration: + break + if out: + bufs.append(out) + else: + break + leng += len(out) + out = b"".join(bufs) + if num >= 0: + self.leftover = out[num:] + out = out[:num] + else: + self.leftover = b"" + self.loc += len(out) + return out + + def close(self): + self.r.close() + self.closed = True + + +def get_range(session, url, start, end, **kwargs): + # explicit get a range when we know it must be safe + kwargs = kwargs.copy() + headers = kwargs.pop("headers", {}).copy() + headers["Range"] = f"bytes={start}-{end - 1}" + r = session.get(url, headers=headers, **kwargs) + r.raise_for_status() + return r.content + + +def _file_info(url, session, size_policy="head", **kwargs): + """Call HEAD on the server to get details about the file (size/checksum etc.) + + Default operation is to explicitly allow redirects and use encoding + 'identity' (no compression) to get the true size of the target. + """ + logger.debug("Retrieve file size for %s", url) + kwargs = kwargs.copy() + ar = kwargs.pop("allow_redirects", True) + head = kwargs.get("headers", {}).copy() + # TODO: not allowed in JS + # head["Accept-Encoding"] = "identity" + kwargs["headers"] = head + + info = {} + if size_policy == "head": + r = session.head(url, allow_redirects=ar, **kwargs) + elif size_policy == "get": + r = session.get(url, allow_redirects=ar, **kwargs) + else: + raise TypeError(f'size_policy must be "head" or "get", got {size_policy}') + r.raise_for_status() + + # TODO: + # recognise lack of 'Accept-Ranges', + # or 'Accept-Ranges': 'none' (not 'bytes') + # to mean streaming only, no random access => return None + if "Content-Length" in r.headers: + info["size"] = int(r.headers["Content-Length"]) + elif "Content-Range" in r.headers: + info["size"] = int(r.headers["Content-Range"].split("/")[1]) + elif "content-length" in r.headers: + info["size"] = int(r.headers["content-length"]) + elif "content-range" in r.headers: + info["size"] = int(r.headers["content-range"].split("/")[1]) + + for checksum_field in ["ETag", "Content-MD5", "Digest"]: + if r.headers.get(checksum_field): + info[checksum_field] = r.headers[checksum_field] + + return info + + +# importing this is enough to register it +def register(): + register_implementation("http", HTTPFileSystem, clobber=True) + register_implementation("https", HTTPFileSystem, clobber=True) + register_implementation("sync_http", HTTPFileSystem, clobber=True) + register_implementation("sync_https", HTTPFileSystem, clobber=True) + + +register() + + +def unregister(): + from fsspec.implementations.http import HTTPFileSystem + + register_implementation("http", HTTPFileSystem, clobber=True) + register_implementation("https", HTTPFileSystem, clobber=True) diff --git a/lib/python3.12/site-packages/fsspec/implementations/jupyter.py b/lib/python3.12/site-packages/fsspec/implementations/jupyter.py new file mode 100644 index 0000000000000000000000000000000000000000..2839f4c1feea56dddd54bdc00f0b884c8461d29e --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/jupyter.py @@ -0,0 +1,124 @@ +import base64 +import io +import re + +import requests + +import fsspec + + +class JupyterFileSystem(fsspec.AbstractFileSystem): + """View of the files as seen by a Jupyter server (notebook or lab)""" + + protocol = ("jupyter", "jlab") + + def __init__(self, url, tok=None, **kwargs): + """ + + Parameters + ---------- + url : str + Base URL of the server, like "http://127.0.0.1:8888". May include + token in the string, which is given by the process when starting up + tok : str + If the token is obtained separately, can be given here + kwargs + """ + if "?" in url: + if tok is None: + try: + tok = re.findall("token=([a-z0-9]+)", url)[0] + except IndexError as e: + raise ValueError("Could not determine token") from e + url = url.split("?", 1)[0] + self.url = url.rstrip("/") + "/api/contents" + self.session = requests.Session() + if tok: + self.session.headers["Authorization"] = f"token {tok}" + + super().__init__(**kwargs) + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + r = self.session.get(f"{self.url}/{path}") + if r.status_code == 404: + return FileNotFoundError(path) + r.raise_for_status() + out = r.json() + + if out["type"] == "directory": + out = out["content"] + else: + out = [out] + for o in out: + o["name"] = o.pop("path") + o.pop("content") + if o["type"] == "notebook": + o["type"] = "file" + if detail: + return out + return [o["name"] for o in out] + + def cat_file(self, path, start=None, end=None, **kwargs): + path = self._strip_protocol(path) + r = self.session.get(f"{self.url}/{path}") + if r.status_code == 404: + return FileNotFoundError(path) + r.raise_for_status() + out = r.json() + if out["format"] == "text": + # data should be binary + b = out["content"].encode() + else: + b = base64.b64decode(out["content"]) + return b[start:end] + + def pipe_file(self, path, value, **_): + path = self._strip_protocol(path) + json = { + "name": path.rsplit("/", 1)[-1], + "path": path, + "size": len(value), + "content": base64.b64encode(value).decode(), + "format": "base64", + "type": "file", + } + self.session.put(f"{self.url}/{path}", json=json) + + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if create_parents and "/" in path: + self.mkdir(path.rsplit("/", 1)[0], True) + json = { + "name": path.rsplit("/", 1)[-1], + "path": path, + "size": None, + "content": None, + "type": "directory", + } + self.session.put(f"{self.url}/{path}", json=json) + + def _rm(self, path): + path = self._strip_protocol(path) + self.session.delete(f"{self.url}/{path}") + + def _open(self, path, mode="rb", **kwargs): + path = self._strip_protocol(path) + if mode == "rb": + data = self.cat_file(path) + return io.BytesIO(data) + else: + return SimpleFileWriter(self, path, mode="wb") + + +class SimpleFileWriter(fsspec.spec.AbstractBufferedFile): + def _upload_chunk(self, final=False): + """Never uploads a chunk until file is done + + Not suitable for large files + """ + if final is False: + return False + self.buffer.seek(0) + data = self.buffer.read() + self.fs.pipe_file(self.path, data) diff --git a/lib/python3.12/site-packages/fsspec/implementations/libarchive.py b/lib/python3.12/site-packages/fsspec/implementations/libarchive.py new file mode 100644 index 0000000000000000000000000000000000000000..eb6f145352e1989e0477e259be02d8d7f4d729e2 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/libarchive.py @@ -0,0 +1,213 @@ +from contextlib import contextmanager +from ctypes import ( + CFUNCTYPE, + POINTER, + c_int, + c_longlong, + c_void_p, + cast, + create_string_buffer, +) + +import libarchive +import libarchive.ffi as ffi + +from fsspec import open_files +from fsspec.archive import AbstractArchiveFileSystem +from fsspec.implementations.memory import MemoryFile +from fsspec.utils import DEFAULT_BLOCK_SIZE + +# Libarchive requires seekable files or memory only for certain archive +# types. However, since we read the directory first to cache the contents +# and also allow random access to any file, the file-like object needs +# to be seekable no matter what. + +# Seek call-backs (not provided in the libarchive python wrapper) +SEEK_CALLBACK = CFUNCTYPE(c_longlong, c_int, c_void_p, c_longlong, c_int) +read_set_seek_callback = ffi.ffi( + "read_set_seek_callback", [ffi.c_archive_p, SEEK_CALLBACK], c_int, ffi.check_int +) +new_api = hasattr(ffi, "NO_OPEN_CB") + + +@contextmanager +def custom_reader(file, format_name="all", filter_name="all", block_size=ffi.page_size): + """Read an archive from a seekable file-like object. + + The `file` object must support the standard `readinto` and 'seek' methods. + """ + buf = create_string_buffer(block_size) + buf_p = cast(buf, c_void_p) + + def read_func(archive_p, context, ptrptr): + # readinto the buffer, returns number of bytes read + length = file.readinto(buf) + # write the address of the buffer into the pointer + ptrptr = cast(ptrptr, POINTER(c_void_p)) + ptrptr[0] = buf_p + # tell libarchive how much data was written into the buffer + return length + + def seek_func(archive_p, context, offset, whence): + file.seek(offset, whence) + # tell libarchvie the current position + return file.tell() + + read_cb = ffi.READ_CALLBACK(read_func) + seek_cb = SEEK_CALLBACK(seek_func) + + if new_api: + open_cb = ffi.NO_OPEN_CB + close_cb = ffi.NO_CLOSE_CB + else: + open_cb = libarchive.read.OPEN_CALLBACK(ffi.VOID_CB) + close_cb = libarchive.read.CLOSE_CALLBACK(ffi.VOID_CB) + + with libarchive.read.new_archive_read(format_name, filter_name) as archive_p: + read_set_seek_callback(archive_p, seek_cb) + ffi.read_open(archive_p, None, open_cb, read_cb, close_cb) + yield libarchive.read.ArchiveRead(archive_p) + + +class LibArchiveFileSystem(AbstractArchiveFileSystem): + """Compressed archives as a file-system (read-only) + + Supports the following formats: + tar, pax , cpio, ISO9660, zip, mtree, shar, ar, raw, xar, lha/lzh, rar + Microsoft CAB, 7-Zip, WARC + + See the libarchive documentation for further restrictions. + https://www.libarchive.org/ + + Keeps file object open while instance lives. It only works in seekable + file-like objects. In case the filesystem does not support this kind of + file object, it is recommended to cache locally. + + This class is pickleable, but not necessarily thread-safe (depends on the + platform). See libarchive documentation for details. + """ + + root_marker = "" + protocol = "libarchive" + cachable = False + + def __init__( + self, + fo="", + mode="r", + target_protocol=None, + target_options=None, + block_size=DEFAULT_BLOCK_SIZE, + **kwargs, + ): + """ + Parameters + ---------- + fo: str or file-like + Contains ZIP, and must exist. If a str, will fetch file using + :meth:`~fsspec.open_files`, which must return one file exactly. + mode: str + Currently, only 'r' accepted + target_protocol: str (optional) + If ``fo`` is a string, this value can be used to override the + FS protocol inferred from a URL + target_options: dict (optional) + Kwargs passed when instantiating the target FS, if ``fo`` is + a string. + """ + super().__init__(self, **kwargs) + if mode != "r": + raise ValueError("Only read from archive files accepted") + if isinstance(fo, str): + files = open_files(fo, protocol=target_protocol, **(target_options or {})) + if len(files) != 1: + raise ValueError( + f'Path "{fo}" did not resolve to exactly one file: "{files}"' + ) + fo = files[0] + self.of = fo + self.fo = fo.__enter__() # the whole instance is a context + self.block_size = block_size + self.dir_cache = None + + @contextmanager + def _open_archive(self): + self.fo.seek(0) + with custom_reader(self.fo, block_size=self.block_size) as arc: + yield arc + + @classmethod + def _strip_protocol(cls, path): + # file paths are always relative to the archive root + return super()._strip_protocol(path).lstrip("/") + + def _get_dirs(self): + fields = { + "name": "pathname", + "size": "size", + "created": "ctime", + "mode": "mode", + "uid": "uid", + "gid": "gid", + "mtime": "mtime", + } + + if self.dir_cache is not None: + return + + self.dir_cache = {} + list_names = [] + with self._open_archive() as arc: + for entry in arc: + if not entry.isdir and not entry.isfile: + # Skip symbolic links, fifo entries, etc. + continue + self.dir_cache.update( + { + dirname: {"name": dirname, "size": 0, "type": "directory"} + for dirname in self._all_dirnames(set(entry.name)) + } + ) + f = {key: getattr(entry, fields[key]) for key in fields} + f["type"] = "directory" if entry.isdir else "file" + list_names.append(entry.name) + + self.dir_cache[f["name"]] = f + # libarchive does not seem to return an entry for the directories (at least + # not in all formats), so get the directories names from the files names + self.dir_cache.update( + { + dirname: {"name": dirname, "size": 0, "type": "directory"} + for dirname in self._all_dirnames(list_names) + } + ) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if mode != "rb": + raise NotImplementedError + + data = bytes() + with self._open_archive() as arc: + for entry in arc: + if entry.pathname != path: + continue + + if entry.size == 0: + # empty file, so there are no blocks + break + + for block in entry.get_blocks(entry.size): + data = block + break + else: + raise ValueError + return MemoryFile(fs=self, path=path, data=data) diff --git a/lib/python3.12/site-packages/fsspec/implementations/local.py b/lib/python3.12/site-packages/fsspec/implementations/local.py new file mode 100644 index 0000000000000000000000000000000000000000..619ea1d0089fbe6ad3eede126fa3a253f446fe14 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/local.py @@ -0,0 +1,477 @@ +import datetime +import io +import logging +import os +import os.path as osp +import shutil +import stat +import tempfile + +from fsspec import AbstractFileSystem +from fsspec.compression import compr +from fsspec.core import get_compression +from fsspec.utils import isfilelike, stringify_path + +logger = logging.getLogger("fsspec.local") + + +class LocalFileSystem(AbstractFileSystem): + """Interface to files on local storage + + Parameters + ---------- + auto_mkdir: bool + Whether, when opening a file, the directory containing it should + be created (if it doesn't already exist). This is assumed by pyarrow + code. + """ + + root_marker = "/" + protocol = "file", "local" + local_file = True + + def __init__(self, auto_mkdir=False, **kwargs): + super().__init__(**kwargs) + self.auto_mkdir = auto_mkdir + + @property + def fsid(self): + return "local" + + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if self.exists(path): + raise FileExistsError(path) + if create_parents: + self.makedirs(path, exist_ok=True) + else: + os.mkdir(path, **kwargs) + + def makedirs(self, path, exist_ok=False): + path = self._strip_protocol(path) + os.makedirs(path, exist_ok=exist_ok) + + def rmdir(self, path): + path = self._strip_protocol(path) + os.rmdir(path) + + def ls(self, path, detail=False, **kwargs): + path = self._strip_protocol(path) + path_info = self.info(path) + infos = [] + if path_info["type"] == "directory": + with os.scandir(path) as it: + for f in it: + try: + # Only get the info if requested since it is a bit expensive (the stat call inside) + # The strip_protocol is also used in info() and calls make_path_posix to always return posix paths + info = self.info(f) if detail else self._strip_protocol(f.path) + infos.append(info) + except FileNotFoundError: + pass + else: + infos = [path_info] if detail else [path_info["name"]] + + return infos + + def info(self, path, **kwargs): + if isinstance(path, os.DirEntry): + # scandir DirEntry + out = path.stat(follow_symlinks=False) + link = path.is_symlink() + if path.is_dir(follow_symlinks=False): + t = "directory" + elif path.is_file(follow_symlinks=False): + t = "file" + else: + t = "other" + + size = out.st_size + if link: + try: + out2 = path.stat(follow_symlinks=True) + size = out2.st_size + except OSError: + size = 0 + path = self._strip_protocol(path.path) + else: + # str or path-like + path = self._strip_protocol(path) + out = os.stat(path, follow_symlinks=False) + link = stat.S_ISLNK(out.st_mode) + if link: + out = os.stat(path, follow_symlinks=True) + size = out.st_size + if stat.S_ISDIR(out.st_mode): + t = "directory" + elif stat.S_ISREG(out.st_mode): + t = "file" + else: + t = "other" + result = { + "name": path, + "size": size, + "type": t, + "created": out.st_ctime, + "islink": link, + } + for field in ["mode", "uid", "gid", "mtime", "ino", "nlink"]: + result[field] = getattr(out, f"st_{field}") + if link: + result["destination"] = os.readlink(path) + return result + + def lexists(self, path, **kwargs): + return osp.lexists(path) + + def cp_file(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + if self.auto_mkdir: + self.makedirs(self._parent(path2), exist_ok=True) + if self.isfile(path1): + shutil.copyfile(path1, path2) + elif self.isdir(path1): + self.mkdirs(path2, exist_ok=True) + else: + raise FileNotFoundError(path1) + + def isfile(self, path): + path = self._strip_protocol(path) + return os.path.isfile(path) + + def isdir(self, path): + path = self._strip_protocol(path) + return os.path.isdir(path) + + def get_file(self, path1, path2, callback=None, **kwargs): + if isfilelike(path2): + with open(path1, "rb") as f: + shutil.copyfileobj(f, path2) + else: + return self.cp_file(path1, path2, **kwargs) + + def put_file(self, path1, path2, callback=None, **kwargs): + return self.cp_file(path1, path2, **kwargs) + + def mv(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + shutil.move(path1, path2) + + def link(self, src, dst, **kwargs): + src = self._strip_protocol(src) + dst = self._strip_protocol(dst) + os.link(src, dst, **kwargs) + + def symlink(self, src, dst, **kwargs): + src = self._strip_protocol(src) + dst = self._strip_protocol(dst) + os.symlink(src, dst, **kwargs) + + def islink(self, path) -> bool: + return os.path.islink(self._strip_protocol(path)) + + def rm_file(self, path): + os.remove(self._strip_protocol(path)) + + def rm(self, path, recursive=False, maxdepth=None): + if not isinstance(path, list): + path = [path] + + for p in path: + p = self._strip_protocol(p) + if self.isdir(p): + if not recursive: + raise ValueError("Cannot delete directory, set recursive=True") + if osp.abspath(p) == os.getcwd(): + raise ValueError("Cannot delete current working directory") + shutil.rmtree(p) + else: + os.remove(p) + + def unstrip_protocol(self, name): + name = self._strip_protocol(name) # normalise for local/win/... + return f"file://{name}" + + def _open(self, path, mode="rb", block_size=None, **kwargs): + path = self._strip_protocol(path) + if self.auto_mkdir and "w" in mode: + self.makedirs(self._parent(path), exist_ok=True) + return LocalFileOpener(path, mode, fs=self, **kwargs) + + def touch(self, path, truncate=True, **kwargs): + path = self._strip_protocol(path) + if self.auto_mkdir: + self.makedirs(self._parent(path), exist_ok=True) + if self.exists(path): + os.utime(path, None) + else: + open(path, "a").close() + if truncate: + os.truncate(path, 0) + + def created(self, path): + info = self.info(path=path) + return datetime.datetime.fromtimestamp( + info["created"], tz=datetime.timezone.utc + ) + + def modified(self, path): + info = self.info(path=path) + return datetime.datetime.fromtimestamp(info["mtime"], tz=datetime.timezone.utc) + + @classmethod + def _parent(cls, path): + path = cls._strip_protocol(path) + if os.sep == "/": + # posix native + return path.rsplit("/", 1)[0] or "/" + else: + # NT + path_ = path.rsplit("/", 1)[0] + if len(path_) <= 3: + if path_[1:2] == ":": + # nt root (something like c:/) + return path_[0] + ":/" + # More cases may be required here + return path_ + + @classmethod + def _strip_protocol(cls, path): + path = stringify_path(path) + if path.startswith("file://"): + path = path[7:] + elif path.startswith("file:"): + path = path[5:] + elif path.startswith("local://"): + path = path[8:] + elif path.startswith("local:"): + path = path[6:] + + path = make_path_posix(path) + if os.sep != "/": + # This code-path is a stripped down version of + # > drive, path = ntpath.splitdrive(path) + if path[1:2] == ":": + # Absolute drive-letter path, e.g. X:\Windows + # Relative path with drive, e.g. X:Windows + drive, path = path[:2], path[2:] + elif path[:2] == "//": + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + if (index1 := path.find("/", 2)) == -1 or ( + index2 := path.find("/", index1 + 1) + ) == -1: + drive, path = path, "" + else: + drive, path = path[:index2], path[index2:] + else: + # Relative path, e.g. Windows + drive = "" + + path = path.rstrip("/") or cls.root_marker + return drive + path + + else: + return path.rstrip("/") or cls.root_marker + + def _isfilestore(self): + # Inheriting from DaskFileSystem makes this False (S3, etc. were) + # the original motivation. But we are a posix-like file system. + # See https://github.com/dask/dask/issues/5526 + return True + + def chmod(self, path, mode): + path = stringify_path(path) + return os.chmod(path, mode) + + +def make_path_posix(path): + """Make path generic and absolute for current OS""" + if not isinstance(path, str): + if isinstance(path, (list, set, tuple)): + return type(path)(make_path_posix(p) for p in path) + else: + path = stringify_path(path) + if not isinstance(path, str): + raise TypeError(f"could not convert {path!r} to string") + if os.sep == "/": + # Native posix + if path.startswith("/"): + # most common fast case for posix + return path + elif path.startswith("~"): + return osp.expanduser(path) + elif path.startswith("./"): + path = path[2:] + elif path == ".": + path = "" + return f"{os.getcwd()}/{path}" + else: + # NT handling + if path[0:1] == "/" and path[2:3] == ":": + # path is like "/c:/local/path" + path = path[1:] + if path[1:2] == ":": + # windows full path like "C:\\local\\path" + if len(path) <= 3: + # nt root (something like c:/) + return path[0] + ":/" + path = path.replace("\\", "/") + return path + elif path[0:1] == "~": + return make_path_posix(osp.expanduser(path)) + elif path.startswith(("\\\\", "//")): + # windows UNC/DFS-style paths + return "//" + path[2:].replace("\\", "/") + elif path.startswith(("\\", "/")): + # windows relative path with root + path = path.replace("\\", "/") + return f"{osp.splitdrive(os.getcwd())[0]}{path}" + else: + path = path.replace("\\", "/") + if path.startswith("./"): + path = path[2:] + elif path == ".": + path = "" + return f"{make_path_posix(os.getcwd())}/{path}" + + +def trailing_sep(path): + """Return True if the path ends with a path separator. + + A forward slash is always considered a path separator, even on Operating + Systems that normally use a backslash. + """ + # TODO: if all incoming paths were posix-compliant then separator would + # always be a forward slash, simplifying this function. + # See https://github.com/fsspec/filesystem_spec/pull/1250 + return path.endswith(os.sep) or (os.altsep is not None and path.endswith(os.altsep)) + + +class LocalFileOpener(io.IOBase): + def __init__( + self, path, mode, autocommit=True, fs=None, compression=None, **kwargs + ): + logger.debug("open file: %s", path) + self.path = path + self.mode = mode + self.fs = fs + self.f = None + self.autocommit = autocommit + self.compression = get_compression(path, compression) + self.blocksize = io.DEFAULT_BUFFER_SIZE + self._open() + + def _open(self): + if self.f is None or self.f.closed: + if self.autocommit or "w" not in self.mode: + self.f = open(self.path, mode=self.mode) + if self.compression: + compress = compr[self.compression] + self.f = compress(self.f, mode=self.mode) + else: + # TODO: check if path is writable? + i, name = tempfile.mkstemp() + os.close(i) # we want normal open and normal buffered file + self.temp = name + self.f = open(name, mode=self.mode) + if "w" not in self.mode: + self.size = self.f.seek(0, 2) + self.f.seek(0) + self.f.size = self.size + + def _fetch_range(self, start, end): + # probably only used by cached FS + if "r" not in self.mode: + raise ValueError + self._open() + self.f.seek(start) + return self.f.read(end - start) + + def __setstate__(self, state): + self.f = None + loc = state.pop("loc", None) + self.__dict__.update(state) + if "r" in state["mode"]: + self.f = None + self._open() + self.f.seek(loc) + + def __getstate__(self): + d = self.__dict__.copy() + d.pop("f") + if "r" in self.mode: + d["loc"] = self.f.tell() + else: + if not self.f.closed: + raise ValueError("Cannot serialise open write-mode local file") + return d + + def commit(self): + if self.autocommit: + raise RuntimeError("Can only commit if not already set to autocommit") + shutil.move(self.temp, self.path) + + def discard(self): + if self.autocommit: + raise RuntimeError("Cannot discard if set to autocommit") + os.remove(self.temp) + + def readable(self) -> bool: + return True + + def writable(self) -> bool: + return "r" not in self.mode + + def read(self, *args, **kwargs): + return self.f.read(*args, **kwargs) + + def write(self, *args, **kwargs): + return self.f.write(*args, **kwargs) + + def tell(self, *args, **kwargs): + return self.f.tell(*args, **kwargs) + + def seek(self, *args, **kwargs): + return self.f.seek(*args, **kwargs) + + def seekable(self, *args, **kwargs): + return self.f.seekable(*args, **kwargs) + + def readline(self, *args, **kwargs): + return self.f.readline(*args, **kwargs) + + def readlines(self, *args, **kwargs): + return self.f.readlines(*args, **kwargs) + + def close(self): + return self.f.close() + + def truncate(self, size=None) -> int: + return self.f.truncate(size) + + @property + def closed(self): + return self.f.closed + + def fileno(self): + return self.raw.fileno() + + def flush(self) -> None: + self.f.flush() + + def __iter__(self): + return self.f.__iter__() + + def __getattr__(self, item): + return getattr(self.f, item) + + def __enter__(self): + self._incontext = True + return self + + def __exit__(self, exc_type, exc_value, traceback): + self._incontext = False + self.f.__exit__(exc_type, exc_value, traceback) diff --git a/lib/python3.12/site-packages/fsspec/implementations/memory.py b/lib/python3.12/site-packages/fsspec/implementations/memory.py new file mode 100644 index 0000000000000000000000000000000000000000..c1c526b564861b6a7f4fa604304b2db103926b4c --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/memory.py @@ -0,0 +1,312 @@ +from __future__ import annotations + +import logging +from datetime import datetime, timezone +from errno import ENOTEMPTY +from io import BytesIO +from pathlib import PurePath, PureWindowsPath +from typing import Any, ClassVar + +from fsspec import AbstractFileSystem +from fsspec.implementations.local import LocalFileSystem +from fsspec.utils import stringify_path + +logger = logging.getLogger("fsspec.memoryfs") + + +class MemoryFileSystem(AbstractFileSystem): + """A filesystem based on a dict of BytesIO objects + + This is a global filesystem so instances of this class all point to the same + in memory filesystem. + """ + + store: ClassVar[dict[str, Any]] = {} # global, do not overwrite! + pseudo_dirs = [""] # global, do not overwrite! + protocol = "memory" + root_marker = "/" + + @classmethod + def _strip_protocol(cls, path): + if isinstance(path, PurePath): + if isinstance(path, PureWindowsPath): + return LocalFileSystem._strip_protocol(path) + else: + path = stringify_path(path) + + if path.startswith("memory://"): + path = path[len("memory://") :] + if "::" in path or "://" in path: + return path.rstrip("/") + path = path.lstrip("/").rstrip("/") + return "/" + path if path else "" + + def ls(self, path, detail=True, **kwargs): + path = self._strip_protocol(path) + if path in self.store: + # there is a key with this exact name + if not detail: + return [path] + return [ + { + "name": path, + "size": self.store[path].size, + "type": "file", + "created": self.store[path].created.timestamp(), + } + ] + paths = set() + starter = path + "/" + out = [] + for p2 in tuple(self.store): + if p2.startswith(starter): + if "/" not in p2[len(starter) :]: + # exact child + out.append( + { + "name": p2, + "size": self.store[p2].size, + "type": "file", + "created": self.store[p2].created.timestamp(), + } + ) + elif len(p2) > len(starter): + # implied child directory + ppath = starter + p2[len(starter) :].split("/", 1)[0] + if ppath not in paths: + out = out or [] + out.append( + { + "name": ppath, + "size": 0, + "type": "directory", + } + ) + paths.add(ppath) + for p2 in self.pseudo_dirs: + if p2.startswith(starter): + if "/" not in p2[len(starter) :]: + # exact child pdir + if p2 not in paths: + out.append({"name": p2, "size": 0, "type": "directory"}) + paths.add(p2) + else: + # directory implied by deeper pdir + ppath = starter + p2[len(starter) :].split("/", 1)[0] + if ppath not in paths: + out.append({"name": ppath, "size": 0, "type": "directory"}) + paths.add(ppath) + if not out: + if path in self.pseudo_dirs: + # empty dir + return [] + raise FileNotFoundError(path) + if detail: + return out + return sorted([f["name"] for f in out]) + + def mkdir(self, path, create_parents=True, **kwargs): + path = self._strip_protocol(path) + if path in self.store or path in self.pseudo_dirs: + raise FileExistsError(path) + if self._parent(path).strip("/") and self.isfile(self._parent(path)): + raise NotADirectoryError(self._parent(path)) + if create_parents and self._parent(path).strip("/"): + try: + self.mkdir(self._parent(path), create_parents, **kwargs) + except FileExistsError: + pass + if path and path not in self.pseudo_dirs: + self.pseudo_dirs.append(path) + + def makedirs(self, path, exist_ok=False): + try: + self.mkdir(path, create_parents=True) + except FileExistsError: + if not exist_ok: + raise + + def pipe_file(self, path, value, mode="overwrite", **kwargs): + """Set the bytes of given file + + Avoids copies of the data if possible + """ + mode = "xb" if mode == "create" else "wb" + self.open(path, mode=mode, data=value) + + def rmdir(self, path): + path = self._strip_protocol(path) + if path == "": + # silently avoid deleting FS root + return + if path in self.pseudo_dirs: + if not self.ls(path): + self.pseudo_dirs.remove(path) + else: + raise OSError(ENOTEMPTY, "Directory not empty", path) + else: + raise FileNotFoundError(path) + + def info(self, path, **kwargs): + logger.debug("info: %s", path) + path = self._strip_protocol(path) + if path in self.pseudo_dirs or any( + p.startswith(path + "/") for p in list(self.store) + self.pseudo_dirs + ): + return { + "name": path, + "size": 0, + "type": "directory", + } + elif path in self.store: + filelike = self.store[path] + return { + "name": path, + "size": filelike.size, + "type": "file", + "created": getattr(filelike, "created", None), + } + else: + raise FileNotFoundError(path) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if "x" in mode and self.exists(path): + raise FileExistsError + if path in self.pseudo_dirs: + raise IsADirectoryError(path) + parent = path + while len(parent) > 1: + parent = self._parent(parent) + if self.isfile(parent): + raise FileExistsError(parent) + if mode in ["rb", "ab", "r+b"]: + if path in self.store: + f = self.store[path] + if mode == "ab": + # position at the end of file + f.seek(0, 2) + else: + # position at the beginning of file + f.seek(0) + return f + else: + raise FileNotFoundError(path) + elif mode in {"wb", "xb"}: + if mode == "xb" and self.exists(path): + raise FileExistsError + m = MemoryFile(self, path, kwargs.get("data")) + if not self._intrans: + m.commit() + return m + else: + name = self.__class__.__name__ + raise ValueError(f"unsupported file mode for {name}: {mode!r}") + + def cp_file(self, path1, path2, **kwargs): + path1 = self._strip_protocol(path1) + path2 = self._strip_protocol(path2) + if self.isfile(path1): + self.store[path2] = MemoryFile( + self, path2, self.store[path1].getvalue() + ) # implicit copy + elif self.isdir(path1): + if path2 not in self.pseudo_dirs: + self.pseudo_dirs.append(path2) + else: + raise FileNotFoundError(path1) + + def cat_file(self, path, start=None, end=None, **kwargs): + logger.debug("cat: %s", path) + path = self._strip_protocol(path) + try: + return bytes(self.store[path].getbuffer()[start:end]) + except KeyError as e: + raise FileNotFoundError(path) from e + + def _rm(self, path): + path = self._strip_protocol(path) + try: + del self.store[path] + except KeyError as e: + raise FileNotFoundError(path) from e + + def modified(self, path): + path = self._strip_protocol(path) + try: + return self.store[path].modified + except KeyError as e: + raise FileNotFoundError(path) from e + + def created(self, path): + path = self._strip_protocol(path) + try: + return self.store[path].created + except KeyError as e: + raise FileNotFoundError(path) from e + + def isfile(self, path): + path = self._strip_protocol(path) + return path in self.store + + def rm(self, path, recursive=False, maxdepth=None): + if isinstance(path, str): + path = self._strip_protocol(path) + else: + path = [self._strip_protocol(p) for p in path] + paths = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) + for p in reversed(paths): + if self.isfile(p): + self.rm_file(p) + # If the expanded path doesn't exist, it is only because the expanded + # path was a directory that does not exist in self.pseudo_dirs. This + # is possible if you directly create files without making the + # directories first. + elif not self.exists(p): + continue + else: + self.rmdir(p) + + +class MemoryFile(BytesIO): + """A BytesIO which can't close and works as a context manager + + Can initialise with data. Each path should only be active once at any moment. + + No need to provide fs, path if auto-committing (default) + """ + + def __init__(self, fs=None, path=None, data=None): + logger.debug("open file %s", path) + self.fs = fs + self.path = path + self.created = datetime.now(tz=timezone.utc) + self.modified = datetime.now(tz=timezone.utc) + if data: + super().__init__(data) + self.seek(0) + + @property + def size(self): + return self.getbuffer().nbytes + + def __enter__(self): + return self + + def close(self): + pass + + def discard(self): + pass + + def commit(self): + self.fs.store[self.path] = self + self.modified = datetime.now(tz=timezone.utc) diff --git a/lib/python3.12/site-packages/fsspec/implementations/reference.py b/lib/python3.12/site-packages/fsspec/implementations/reference.py new file mode 100644 index 0000000000000000000000000000000000000000..35da812c68303b706022ae4d48698e9621eab63f --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/reference.py @@ -0,0 +1,1305 @@ +import base64 +import collections +import io +import itertools +import logging +import math +import os +from functools import lru_cache +from itertools import chain +from typing import TYPE_CHECKING, Literal + +import fsspec.core +from fsspec.spec import AbstractBufferedFile + +try: + import ujson as json +except ImportError: + if not TYPE_CHECKING: + import json + +from fsspec.asyn import AsyncFileSystem +from fsspec.callbacks import DEFAULT_CALLBACK +from fsspec.core import filesystem, open, split_protocol +from fsspec.implementations.asyn_wrapper import AsyncFileSystemWrapper +from fsspec.utils import isfilelike, merge_offset_ranges, other_paths + +logger = logging.getLogger("fsspec.reference") + + +class ReferenceNotReachable(RuntimeError): + def __init__(self, reference, target, *args): + super().__init__(*args) + self.reference = reference + self.target = target + + def __str__(self): + return f'Reference "{self.reference}" failed to fetch target {self.target}' + + +def _first(d): + return next(iter(d.values())) + + +def _prot_in_references(path, references): + ref = references.get(path) + if isinstance(ref, (list, tuple)) and isinstance(ref[0], str): + return split_protocol(ref[0])[0] if ref[0] else ref[0] + + +def _protocol_groups(paths, references): + if isinstance(paths, str): + return {_prot_in_references(paths, references): [paths]} + out = {} + for path in paths: + protocol = _prot_in_references(path, references) + out.setdefault(protocol, []).append(path) + return out + + +class RefsValuesView(collections.abc.ValuesView): + def __iter__(self): + for val in self._mapping.zmetadata.values(): + yield json.dumps(val).encode() + yield from self._mapping._items.values() + for field in self._mapping.listdir(): + chunk_sizes = self._mapping._get_chunk_sizes(field) + if len(chunk_sizes) == 0: + yield self._mapping[field + "/0"] + continue + yield from self._mapping._generate_all_records(field) + + +class RefsItemsView(collections.abc.ItemsView): + def __iter__(self): + return zip(self._mapping.keys(), self._mapping.values()) + + +def ravel_multi_index(idx, sizes): + val = 0 + mult = 1 + for i, s in zip(idx[::-1], sizes[::-1]): + val += i * mult + mult *= s + return val + + +class LazyReferenceMapper(collections.abc.MutableMapping): + """This interface can be used to read/write references from Parquet stores. + It is not intended for other types of references. + It can be used with Kerchunk's MultiZarrToZarr method to combine + references into a parquet store. + Examples of this use-case can be found here: + https://fsspec.github.io/kerchunk/advanced.html?highlight=parquet#parquet-storage""" + + # import is class level to prevent numpy dep requirement for fsspec + @property + def np(self): + import numpy as np + + return np + + @property + def pd(self): + import pandas as pd + + return pd + + def __init__( + self, + root, + fs=None, + out_root=None, + cache_size=128, + categorical_threshold=10, + engine: Literal["fastparquet", "pyarrow"] = "fastparquet", + ): + """ + + This instance will be writable, storing changes in memory until full partitions + are accumulated or .flush() is called. + + To create an empty lazy store, use .create() + + Parameters + ---------- + root : str + Root of parquet store + fs : fsspec.AbstractFileSystem + fsspec filesystem object, default is local filesystem. + cache_size : int, default=128 + Maximum size of LRU cache, where cache_size*record_size denotes + the total number of references that can be loaded in memory at once. + categorical_threshold : int + Encode urls as pandas.Categorical to reduce memory footprint if the ratio + of the number of unique urls to total number of refs for each variable + is greater than or equal to this number. (default 10) + engine: Literal["fastparquet","pyarrow"] + Engine choice for reading parquet files. (default is "fastparquet") + """ + + self.root = root + self.chunk_sizes = {} + self.cat_thresh = categorical_threshold + self.engine = engine + self.cache_size = cache_size + self.url = self.root + "/{field}/refs.{record}.parq" + # TODO: derive fs from `root` + self.fs = fsspec.filesystem("file") if fs is None else fs + self.out_root = self.fs.unstrip_protocol(out_root or self.root) + + from importlib.util import find_spec + + if self.engine == "pyarrow" and find_spec("pyarrow") is None: + raise ImportError("engine choice `pyarrow` is not installed.") + + def __getattr__(self, item): + if item in ("_items", "record_size", "zmetadata"): + self.setup() + # avoid possible recursion if setup fails somehow + return self.__dict__[item] + raise AttributeError(item) + + def setup(self): + self._items = {} + self._items[".zmetadata"] = self.fs.cat_file( + "/".join([self.root, ".zmetadata"]) + ) + met = json.loads(self._items[".zmetadata"]) + self.record_size = met["record_size"] + self.zmetadata = met["metadata"] + + # Define function to open and decompress refs + @lru_cache(maxsize=self.cache_size) + def open_refs(field, record): + """cached parquet file loader""" + path = self.url.format(field=field, record=record) + data = io.BytesIO(self.fs.cat_file(path)) + try: + df = self.pd.read_parquet(data, engine=self.engine) + refs = {c: df[c].to_numpy() for c in df.columns} + except OSError: + refs = None + return refs + + self.open_refs = open_refs + + @staticmethod + def create(root, storage_options=None, fs=None, record_size=10000, **kwargs): + """Make empty parquet reference set + + First deletes the contents of the given directory, if it exists. + + Parameters + ---------- + root: str + Directory to contain the output; will be created + storage_options: dict | None + For making the filesystem to use for writing is fs is None + fs: FileSystem | None + Filesystem for writing + record_size: int + Number of references per parquet file + kwargs: passed to __init__ + + Returns + ------- + LazyReferenceMapper instance + """ + met = {"metadata": {}, "record_size": record_size} + if fs is None: + fs, root = fsspec.core.url_to_fs(root, **(storage_options or {})) + if fs.exists(root): + fs.rm(root, recursive=True) + fs.makedirs(root, exist_ok=True) + fs.pipe("/".join([root, ".zmetadata"]), json.dumps(met).encode()) + return LazyReferenceMapper(root, fs, **kwargs) + + @lru_cache() + def listdir(self): + """List top-level directories""" + dirs = (p.rsplit("/", 1)[0] for p in self.zmetadata if not p.startswith(".z")) + return set(dirs) + + def ls(self, path="", detail=True): + """Shortcut file listings""" + path = path.rstrip("/") + pathdash = path + "/" if path else "" + dirnames = self.listdir() + dirs = [ + d + for d in dirnames + if d.startswith(pathdash) and "/" not in d.lstrip(pathdash) + ] + if dirs: + others = { + f + for f in chain( + [".zmetadata"], + (name for name in self.zmetadata), + (name for name in self._items), + ) + if f.startswith(pathdash) and "/" not in f.lstrip(pathdash) + } + if detail is False: + others.update(dirs) + return sorted(others) + dirinfo = [{"name": name, "type": "directory", "size": 0} for name in dirs] + fileinfo = [ + { + "name": name, + "type": "file", + "size": len( + json.dumps(self.zmetadata[name]) + if name in self.zmetadata + else self._items[name] + ), + } + for name in others + ] + return sorted(dirinfo + fileinfo, key=lambda s: s["name"]) + field = path + others = set( + [name for name in self.zmetadata if name.startswith(f"{path}/")] + + [name for name in self._items if name.startswith(f"{path}/")] + ) + fileinfo = [ + { + "name": name, + "type": "file", + "size": len( + json.dumps(self.zmetadata[name]) + if name in self.zmetadata + else self._items[name] + ), + } + for name in others + ] + keys = self._keys_in_field(field) + + if detail is False: + return list(others) + list(keys) + recs = self._generate_all_records(field) + recinfo = [ + {"name": name, "type": "file", "size": rec[-1]} + for name, rec in zip(keys, recs) + if rec[0] # filters out path==None, deleted/missing + ] + return fileinfo + recinfo + + def _load_one_key(self, key): + """Get the reference for one key + + Returns bytes, one-element list or three-element list. + """ + if key in self._items: + return self._items[key] + elif key in self.zmetadata: + return json.dumps(self.zmetadata[key]).encode() + elif "/" not in key or self._is_meta(key): + raise KeyError(key) + field, _ = key.rsplit("/", 1) + record, ri, chunk_size = self._key_to_record(key) + maybe = self._items.get((field, record), {}).get(ri, False) + if maybe is None: + # explicitly deleted + raise KeyError + elif maybe: + return maybe + elif chunk_size == 0: + return b"" + + # Chunk keys can be loaded from row group and cached in LRU cache + try: + refs = self.open_refs(field, record) + except (ValueError, TypeError, FileNotFoundError) as exc: + raise KeyError(key) from exc + columns = ["path", "offset", "size", "raw"] + selection = [refs[c][ri] if c in refs else None for c in columns] + raw = selection[-1] + if raw is not None: + return raw + if selection[0] is None: + raise KeyError("This reference does not exist or has been deleted") + if selection[1:3] == [0, 0]: + # URL only + return selection[:1] + # URL, offset, size + return selection[:3] + + @lru_cache(4096) + def _key_to_record(self, key): + """Details needed to construct a reference for one key""" + field, chunk = key.rsplit("/", 1) + chunk_sizes = self._get_chunk_sizes(field) + if len(chunk_sizes) == 0: + return 0, 0, 0 + chunk_idx = [int(c) for c in chunk.split(".")] + chunk_number = ravel_multi_index(chunk_idx, chunk_sizes) + record = chunk_number // self.record_size + ri = chunk_number % self.record_size + return record, ri, len(chunk_sizes) + + def _get_chunk_sizes(self, field): + """The number of chunks along each axis for a given field""" + if field not in self.chunk_sizes: + zarray = self.zmetadata[f"{field}/.zarray"] + size_ratio = [ + math.ceil(s / c) for s, c in zip(zarray["shape"], zarray["chunks"]) + ] + self.chunk_sizes[field] = size_ratio or [1] + return self.chunk_sizes[field] + + def _generate_record(self, field, record): + """The references for a given parquet file of a given field""" + refs = self.open_refs(field, record) + it = iter(zip(*refs.values())) + if len(refs) == 3: + # All urls + return (list(t) for t in it) + elif len(refs) == 1: + # All raws + return refs["raw"] + else: + # Mix of urls and raws + return (list(t[:3]) if not t[3] else t[3] for t in it) + + def _generate_all_records(self, field): + """Load all the references within a field by iterating over the parquet files""" + nrec = 1 + for ch in self._get_chunk_sizes(field): + nrec *= ch + nrec = math.ceil(nrec / self.record_size) + for record in range(nrec): + yield from self._generate_record(field, record) + + def values(self): + return RefsValuesView(self) + + def items(self): + return RefsItemsView(self) + + def __hash__(self): + return id(self) + + def __getitem__(self, key): + return self._load_one_key(key) + + def __setitem__(self, key, value): + if "/" in key and not self._is_meta(key): + field, chunk = key.rsplit("/", 1) + record, i, _ = self._key_to_record(key) + subdict = self._items.setdefault((field, record), {}) + subdict[i] = value + if len(subdict) == self.record_size: + self.write(field, record) + else: + # metadata or top-level + if hasattr(value, "to_bytes"): + val = value.to_bytes().decode() + elif isinstance(value, bytes): + val = value.decode() + else: + val = value + self._items[key] = val + new_value = json.loads(val) + self.zmetadata[key] = {**self.zmetadata.get(key, {}), **new_value} + + @staticmethod + def _is_meta(key): + return key.startswith(".z") or "/.z" in key + + def __delitem__(self, key): + if key in self._items: + del self._items[key] + elif key in self.zmetadata: + del self.zmetadata[key] + else: + if "/" in key and not self._is_meta(key): + field, _ = key.rsplit("/", 1) + record, i, _ = self._key_to_record(key) + subdict = self._items.setdefault((field, record), {}) + subdict[i] = None + if len(subdict) == self.record_size: + self.write(field, record) + else: + # metadata or top-level + self._items[key] = None + + def write(self, field, record, base_url=None, storage_options=None): + # extra requirements if writing + import kerchunk.df + import numpy as np + import pandas as pd + + partition = self._items[(field, record)] + original = False + if len(partition) < self.record_size: + try: + original = self.open_refs(field, record) + except OSError: + pass + + if original: + paths = original["path"] + offsets = original["offset"] + sizes = original["size"] + raws = original["raw"] + else: + paths = np.full(self.record_size, np.nan, dtype="O") + offsets = np.zeros(self.record_size, dtype="int64") + sizes = np.zeros(self.record_size, dtype="int64") + raws = np.full(self.record_size, np.nan, dtype="O") + for j, data in partition.items(): + if isinstance(data, list): + if ( + str(paths.dtype) == "category" + and data[0] not in paths.dtype.categories + ): + paths = paths.add_categories(data[0]) + paths[j] = data[0] + if len(data) > 1: + offsets[j] = data[1] + sizes[j] = data[2] + elif data is None: + # delete + paths[j] = None + offsets[j] = 0 + sizes[j] = 0 + raws[j] = None + else: + # this is the only call into kerchunk, could remove + raws[j] = kerchunk.df._proc_raw(data) + # TODO: only save needed columns + df = pd.DataFrame( + { + "path": paths, + "offset": offsets, + "size": sizes, + "raw": raws, + }, + copy=False, + ) + if df.path.count() / (df.path.nunique() or 1) > self.cat_thresh: + df["path"] = df["path"].astype("category") + object_encoding = {"raw": "bytes", "path": "utf8"} + has_nulls = ["path", "raw"] + + fn = f"{base_url or self.out_root}/{field}/refs.{record}.parq" + self.fs.mkdirs(f"{base_url or self.out_root}/{field}", exist_ok=True) + + if self.engine == "pyarrow": + df_backend_kwargs = {"write_statistics": False} + elif self.engine == "fastparquet": + df_backend_kwargs = { + "stats": False, + "object_encoding": object_encoding, + "has_nulls": has_nulls, + } + else: + raise NotImplementedError(f"{self.engine} not supported") + df.to_parquet( + fn, + engine=self.engine, + storage_options=storage_options + or getattr(self.fs, "storage_options", None), + compression="zstd", + index=False, + **df_backend_kwargs, + ) + + partition.clear() + self._items.pop((field, record)) + + def flush(self, base_url=None, storage_options=None): + """Output any modified or deleted keys + + Parameters + ---------- + base_url: str + Location of the output + """ + + # write what we have so far and clear sub chunks + for thing in list(self._items): + if isinstance(thing, tuple): + field, record = thing + self.write( + field, + record, + base_url=base_url, + storage_options=storage_options, + ) + + # gather .zmetadata from self._items and write that too + for k in list(self._items): + if k != ".zmetadata" and ".z" in k: + self.zmetadata[k] = json.loads(self._items.pop(k)) + met = {"metadata": self.zmetadata, "record_size": self.record_size} + self._items.clear() + self._items[".zmetadata"] = json.dumps(met).encode() + self.fs.pipe( + "/".join([base_url or self.out_root, ".zmetadata"]), + self._items[".zmetadata"], + ) + + # TODO: only clear those that we wrote to? + self.open_refs.cache_clear() + + def __len__(self): + # Caveat: This counts expected references, not actual - but is fast + count = 0 + for field in self.listdir(): + if field.startswith("."): + count += 1 + else: + count += math.prod(self._get_chunk_sizes(field)) + count += len(self.zmetadata) # all metadata keys + # any other files not in reference partitions + count += sum(1 for _ in self._items if not isinstance(_, tuple)) + return count + + def __iter__(self): + # Caveat: returns only existing keys, so the number of these does not + # match len(self) + metas = set(self.zmetadata) + metas.update(self._items) + for bit in metas: + if isinstance(bit, str): + yield bit + for field in self.listdir(): + for k in self._keys_in_field(field): + if k in self: + yield k + + def __contains__(self, item): + try: + self._load_one_key(item) + return True + except KeyError: + return False + + def _keys_in_field(self, field): + """List key names in given field + + Produces strings like "field/x.y" appropriate from the chunking of the array + """ + chunk_sizes = self._get_chunk_sizes(field) + if len(chunk_sizes) == 0: + yield field + "/0" + return + inds = itertools.product(*(range(i) for i in chunk_sizes)) + for ind in inds: + yield field + "/" + ".".join([str(c) for c in ind]) + + +class ReferenceFileSystem(AsyncFileSystem): + """View byte ranges of some other file as a file system + Initial version: single file system target, which must support + async, and must allow start and end args in _cat_file. Later versions + may allow multiple arbitrary URLs for the targets. + This FileSystem is read-only. It is designed to be used with async + targets (for now). We do not get original file details from the target FS. + Configuration is by passing a dict of references at init, or a URL to + a JSON file containing the same; this dict + can also contain concrete data for some set of paths. + Reference dict format: + {path0: bytes_data, path1: (target_url, offset, size)} + https://github.com/fsspec/kerchunk/blob/main/README.md + """ + + protocol = "reference" + cachable = False + + def __init__( + self, + fo, + target=None, + ref_storage_args=None, + target_protocol=None, + target_options=None, + remote_protocol=None, + remote_options=None, + fs=None, + template_overrides=None, + simple_templates=True, + max_gap=64_000, + max_block=256_000_000, + cache_size=128, + **kwargs, + ): + """ + Parameters + ---------- + fo : dict or str + The set of references to use for this instance, with a structure as above. + If str referencing a JSON file, will use fsspec.open, in conjunction + with target_options and target_protocol to open and parse JSON at this + location. If a directory, then assume references are a set of parquet + files to be loaded lazily. + target : str + For any references having target_url as None, this is the default file + target to use + ref_storage_args : dict + If references is a str, use these kwargs for loading the JSON file. + Deprecated: use target_options instead. + target_protocol : str + Used for loading the reference file, if it is a path. If None, protocol + will be derived from the given path + target_options : dict + Extra FS options for loading the reference file ``fo``, if given as a path + remote_protocol : str + The protocol of the filesystem on which the references will be evaluated + (unless fs is provided). If not given, will be derived from the first + URL that has a protocol in the templates or in the references, in that + order. + remote_options : dict + kwargs to go with remote_protocol + fs : AbstractFileSystem | dict(str, (AbstractFileSystem | dict)) + Directly provide a file system(s): + - a single filesystem instance + - a dict of protocol:filesystem, where each value is either a filesystem + instance, or a dict of kwargs that can be used to create in + instance for the given protocol + + If this is given, remote_options and remote_protocol are ignored. + template_overrides : dict + Swap out any templates in the references file with these - useful for + testing. + simple_templates: bool + Whether templates can be processed with simple replace (True) or if + jinja is needed (False, much slower). All reference sets produced by + ``kerchunk`` are simple in this sense, but the spec allows for complex. + max_gap, max_block: int + For merging multiple concurrent requests to the same remote file. + Neighboring byte ranges will only be merged when their + inter-range gap is <= ``max_gap``. Default is 64KB. Set to 0 + to only merge when it requires no extra bytes. Pass a negative + number to disable merging, appropriate for local target files. + Neighboring byte ranges will only be merged when the size of + the aggregated range is <= ``max_block``. Default is 256MB. + cache_size : int + Maximum size of LRU cache, where cache_size*record_size denotes + the total number of references that can be loaded in memory at once. + Only used for lazily loaded references. + kwargs : passed to parent class + """ + super().__init__(**kwargs) + self.target = target + self.template_overrides = template_overrides + self.simple_templates = simple_templates + self.templates = {} + self.fss = {} + self._dircache = {} + self.max_gap = max_gap + self.max_block = max_block + if isinstance(fo, str): + dic = dict( + **(ref_storage_args or target_options or {}), protocol=target_protocol + ) + ref_fs, fo2 = fsspec.core.url_to_fs(fo, **dic) + if ref_fs.isfile(fo2): + # text JSON + with fsspec.open(fo, "rb", **dic) as f: + logger.info("Read reference from URL %s", fo) + text = json.load(f) + self._process_references(text, template_overrides) + else: + # Lazy parquet refs + logger.info("Open lazy reference dict from URL %s", fo) + self.references = LazyReferenceMapper( + fo2, + fs=ref_fs, + cache_size=cache_size, + ) + else: + # dictionaries + self._process_references(fo, template_overrides) + if isinstance(fs, dict): + self.fss = { + k: ( + fsspec.filesystem(k.split(":", 1)[0], **opts) + if isinstance(opts, dict) + else opts + ) + for k, opts in fs.items() + } + if None not in self.fss: + self.fss[None] = filesystem("file") + return + if fs is not None: + # single remote FS + remote_protocol = ( + fs.protocol[0] if isinstance(fs.protocol, tuple) else fs.protocol + ) + self.fss[remote_protocol] = fs + + if remote_protocol is None: + # get single protocol from any templates + for ref in self.templates.values(): + if callable(ref): + ref = ref() + protocol, _ = fsspec.core.split_protocol(ref) + if protocol and protocol not in self.fss: + fs = filesystem(protocol, **(remote_options or {})) + self.fss[protocol] = fs + if remote_protocol is None: + # get single protocol from references + # TODO: warning here, since this can be very expensive? + for ref in self.references.values(): + if callable(ref): + ref = ref() + if isinstance(ref, list) and ref[0]: + protocol, _ = fsspec.core.split_protocol(ref[0]) + if protocol not in self.fss: + fs = filesystem(protocol, **(remote_options or {})) + self.fss[protocol] = fs + # only use first remote URL + break + + if remote_protocol and remote_protocol not in self.fss: + fs = filesystem(remote_protocol, **(remote_options or {})) + self.fss[remote_protocol] = fs + + self.fss[None] = fs or filesystem("file") # default one + # Wrap any non-async filesystems to ensure async methods are available below + for k, f in self.fss.items(): + if not f.async_impl: + self.fss[k] = AsyncFileSystemWrapper(f, asynchronous=self.asynchronous) + elif self.asynchronous ^ f.asynchronous: + raise ValueError( + "Reference-FS's target filesystem must have same value" + "of asynchronous" + ) + + def _cat_common(self, path, start=None, end=None): + path = self._strip_protocol(path) + logger.debug(f"cat: {path}") + try: + part = self.references[path] + except KeyError as exc: + raise FileNotFoundError(path) from exc + if isinstance(part, str): + part = part.encode() + if hasattr(part, "to_bytes"): + part = part.to_bytes() + if isinstance(part, bytes): + logger.debug(f"Reference: {path}, type bytes") + if part.startswith(b"base64:"): + part = base64.b64decode(part[7:]) + return part, None, None + + if len(part) == 1: + logger.debug(f"Reference: {path}, whole file => {part}") + url = part[0] + start1, end1 = start, end + else: + url, start0, size = part + logger.debug(f"Reference: {path} => {url}, offset {start0}, size {size}") + end0 = start0 + size + + if start is not None: + if start >= 0: + start1 = start0 + start + else: + start1 = end0 + start + else: + start1 = start0 + if end is not None: + if end >= 0: + end1 = start0 + end + else: + end1 = end0 + end + else: + end1 = end0 + if url is None: + url = self.target + return url, start1, end1 + + async def _cat_file(self, path, start=None, end=None, **kwargs): + part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) + if isinstance(part_or_url, bytes): + return part_or_url[start:end] + protocol, _ = split_protocol(part_or_url) + try: + return await self.fss[protocol]._cat_file( + part_or_url, start=start0, end=end0 + ) + except Exception as e: + raise ReferenceNotReachable(path, part_or_url) from e + + def cat_file(self, path, start=None, end=None, **kwargs): + part_or_url, start0, end0 = self._cat_common(path, start=start, end=end) + if isinstance(part_or_url, bytes): + return part_or_url[start:end] + protocol, _ = split_protocol(part_or_url) + try: + return self.fss[protocol].cat_file(part_or_url, start=start0, end=end0) + except Exception as e: + raise ReferenceNotReachable(path, part_or_url) from e + + def pipe_file(self, path, value, **_): + """Temporarily add binary data or reference as a file""" + self.references[path] = value + + async def _get_file(self, rpath, lpath, **kwargs): + if self.isdir(rpath): + return os.makedirs(lpath, exist_ok=True) + data = await self._cat_file(rpath) + with open(lpath, "wb") as f: + f.write(data) + + def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, **kwargs): + if self.isdir(rpath): + return os.makedirs(lpath, exist_ok=True) + data = self.cat_file(rpath, **kwargs) + callback.set_size(len(data)) + if isfilelike(lpath): + lpath.write(data) + else: + with open(lpath, "wb") as f: + f.write(data) + callback.absolute_update(len(data)) + + def get(self, rpath, lpath, recursive=False, **kwargs): + if recursive: + # trigger directory build + self.ls("") + rpath = self.expand_path(rpath, recursive=recursive) + fs = fsspec.filesystem("file", auto_mkdir=True) + targets = other_paths(rpath, lpath) + if recursive: + data = self.cat([r for r in rpath if not self.isdir(r)]) + else: + data = self.cat(rpath) + for remote, local in zip(rpath, targets): + if remote in data: + fs.pipe_file(local, data[remote]) + + def cat(self, path, recursive=False, on_error="raise", **kwargs): + if isinstance(path, str) and recursive: + raise NotImplementedError + if isinstance(path, list) and (recursive or any("*" in p for p in path)): + raise NotImplementedError + # TODO: if references is lazy, pre-fetch all paths in batch before access + proto_dict = _protocol_groups(path, self.references) + out = {} + for proto, paths in proto_dict.items(): + fs = self.fss[proto] + urls, starts, ends, valid_paths = [], [], [], [] + for p in paths: + # find references or label not-found. Early exit if any not + # found and on_error is "raise" + try: + u, s, e = self._cat_common(p) + if not isinstance(u, (bytes, str)): + # nan/None from parquet + continue + except FileNotFoundError as err: + if on_error == "raise": + raise + if on_error != "omit": + out[p] = err + else: + urls.append(u) + starts.append(s) + ends.append(e) + valid_paths.append(p) + + # process references into form for merging + urls2 = [] + starts2 = [] + ends2 = [] + paths2 = [] + whole_files = set() + for u, s, e, p in zip(urls, starts, ends, valid_paths): + if isinstance(u, bytes): + # data + out[p] = u + elif s is None: + # whole file - limits are None, None, but no further + # entries take for this file + whole_files.add(u) + urls2.append(u) + starts2.append(s) + ends2.append(e) + paths2.append(p) + for u, s, e, p in zip(urls, starts, ends, valid_paths): + # second run to account for files that are to be loaded whole + if s is not None and u not in whole_files: + urls2.append(u) + starts2.append(s) + ends2.append(e) + paths2.append(p) + + # merge and fetch consolidated ranges + new_paths, new_starts, new_ends = merge_offset_ranges( + list(urls2), + list(starts2), + list(ends2), + sort=True, + max_gap=self.max_gap, + max_block=self.max_block, + ) + bytes_out = fs.cat_ranges(new_paths, new_starts, new_ends) + + # unbundle from merged bytes - simple approach + for u, s, e, p in zip(urls, starts, ends, valid_paths): + if p in out: + continue # was bytes, already handled + for np, ns, ne, b in zip(new_paths, new_starts, new_ends, bytes_out): + if np == u and (ns is None or ne is None): + if isinstance(b, Exception): + out[p] = b + else: + out[p] = b[s:e] + elif np == u and s >= ns and e <= ne: + if isinstance(b, Exception): + out[p] = b + else: + out[p] = b[s - ns : (e - ne) or None] + + for k, v in out.copy().items(): + # these were valid references, but fetch failed, so transform exc + if isinstance(v, Exception) and k in self.references: + ex = out[k] + new_ex = ReferenceNotReachable(k, self.references[k]) + new_ex.__cause__ = ex + if on_error == "raise": + raise new_ex + elif on_error != "omit": + out[k] = new_ex + + if len(out) == 1 and isinstance(path, str) and "*" not in path: + return _first(out) + return out + + def _process_references(self, references, template_overrides=None): + vers = references.get("version", None) + if vers is None: + self._process_references0(references) + elif vers == 1: + self._process_references1(references, template_overrides=template_overrides) + else: + raise ValueError(f"Unknown reference spec version: {vers}") + # TODO: we make dircache by iterating over all entries, but for Spec >= 1, + # can replace with programmatic. Is it even needed for mapper interface? + + def _process_references0(self, references): + """Make reference dict for Spec Version 0""" + if isinstance(references, dict): + # do not do this for lazy/parquet backend, which will not make dicts, + # but must remain writable in the original object + references = { + key: json.dumps(val) if isinstance(val, dict) else val + for key, val in references.items() + } + self.references = references + + def _process_references1(self, references, template_overrides=None): + if not self.simple_templates or self.templates: + import jinja2 + self.references = {} + self._process_templates(references.get("templates", {})) + + @lru_cache(1000) + def _render_jinja(u): + return jinja2.Template(u).render(**self.templates) + + for k, v in references.get("refs", {}).items(): + if isinstance(v, str): + if v.startswith("base64:"): + self.references[k] = base64.b64decode(v[7:]) + self.references[k] = v + elif isinstance(v, dict): + self.references[k] = json.dumps(v) + elif self.templates: + u = v[0] + if "{{" in u: + if self.simple_templates: + u = ( + u.replace("{{", "{") + .replace("}}", "}") + .format(**self.templates) + ) + else: + u = _render_jinja(u) + self.references[k] = [u] if len(v) == 1 else [u, v[1], v[2]] + else: + self.references[k] = v + self.references.update(self._process_gen(references.get("gen", []))) + + def _process_templates(self, tmp): + self.templates = {} + if self.template_overrides is not None: + tmp.update(self.template_overrides) + for k, v in tmp.items(): + if "{{" in v: + import jinja2 + + self.templates[k] = lambda temp=v, **kwargs: jinja2.Template( + temp + ).render(**kwargs) + else: + self.templates[k] = v + + def _process_gen(self, gens): + out = {} + for gen in gens: + dimension = { + k: ( + v + if isinstance(v, list) + else range(v.get("start", 0), v["stop"], v.get("step", 1)) + ) + for k, v in gen["dimensions"].items() + } + products = ( + dict(zip(dimension.keys(), values)) + for values in itertools.product(*dimension.values()) + ) + for pr in products: + import jinja2 + + key = jinja2.Template(gen["key"]).render(**pr, **self.templates) + url = jinja2.Template(gen["url"]).render(**pr, **self.templates) + if ("offset" in gen) and ("length" in gen): + offset = int( + jinja2.Template(gen["offset"]).render(**pr, **self.templates) + ) + length = int( + jinja2.Template(gen["length"]).render(**pr, **self.templates) + ) + out[key] = [url, offset, length] + elif ("offset" in gen) ^ ("length" in gen): + raise ValueError( + "Both 'offset' and 'length' are required for a " + "reference generator entry if either is provided." + ) + else: + out[key] = [url] + return out + + def _dircache_from_items(self): + self.dircache = {"": []} + it = self.references.items() + for path, part in it: + if isinstance(part, (bytes, str)) or hasattr(part, "to_bytes"): + size = len(part) + elif len(part) == 1: + size = None + else: + _, _, size = part + par = path.rsplit("/", 1)[0] if "/" in path else "" + par0 = par + subdirs = [par0] + while par0 and par0 not in self.dircache: + # collect parent directories + par0 = self._parent(par0) + subdirs.append(par0) + + subdirs.reverse() + for parent, child in zip(subdirs, subdirs[1:]): + # register newly discovered directories + assert child not in self.dircache + assert parent in self.dircache + self.dircache[parent].append( + {"name": child, "type": "directory", "size": 0} + ) + self.dircache[child] = [] + + self.dircache[par].append({"name": path, "type": "file", "size": size}) + + def _open(self, path, mode="rb", block_size=None, cache_options=None, **kwargs): + part_or_url, start0, end0 = self._cat_common(path) + # This logic is kept outside `ReferenceFile` to avoid unnecessary redirection. + # That does mean `_cat_common` gets called twice if it eventually reaches `ReferenceFile`. + if isinstance(part_or_url, bytes): + return io.BytesIO(part_or_url[start0:end0]) + + protocol, _ = split_protocol(part_or_url) + if start0 is None and end0 is None: + return self.fss[protocol]._open( + part_or_url, + mode, + block_size=block_size, + cache_options=cache_options, + **kwargs, + ) + + return ReferenceFile( + self, + path, + mode, + block_size=block_size, + cache_options=cache_options, + **kwargs, + ) + + def ls(self, path, detail=True, **kwargs): + logger.debug("list %s", path) + path = self._strip_protocol(path) + if isinstance(self.references, LazyReferenceMapper): + try: + return self.references.ls(path, detail) + except KeyError: + pass + raise FileNotFoundError(f"'{path}' is not a known key") + if not self.dircache: + self._dircache_from_items() + out = self._ls_from_cache(path) + if out is None: + raise FileNotFoundError(path) + if detail: + return out + return [o["name"] for o in out] + + def exists(self, path, **kwargs): # overwrite auto-sync version + return self.isdir(path) or self.isfile(path) + + def isdir(self, path): # overwrite auto-sync version + if self.dircache: + return path in self.dircache + elif isinstance(self.references, LazyReferenceMapper): + return path in self.references.listdir() + else: + # this may be faster than building dircache for single calls, but + # by looping will be slow for many calls; could cache it? + return any(_.startswith(f"{path}/") for _ in self.references) + + def isfile(self, path): # overwrite auto-sync version + return path in self.references + + async def _ls(self, path, detail=True, **kwargs): # calls fast sync code + return self.ls(path, detail, **kwargs) + + def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + if withdirs: + return super().find( + path, maxdepth=maxdepth, withdirs=withdirs, detail=detail, **kwargs + ) + if path: + path = self._strip_protocol(path) + r = sorted(k for k in self.references if k.startswith(path)) + else: + r = sorted(self.references) + if detail: + if not self.dircache: + self._dircache_from_items() + return {k: self._ls_from_cache(k)[0] for k in r} + else: + return r + + def info(self, path, **kwargs): + out = self.references.get(path) + if out is not None: + if isinstance(out, (str, bytes)): + # decode base64 here + return {"name": path, "type": "file", "size": len(out)} + elif len(out) > 1: + return {"name": path, "type": "file", "size": out[2]} + else: + out0 = [{"name": path, "type": "file", "size": None}] + else: + out = self.ls(path, True) + out0 = [o for o in out if o["name"] == path] + if not out0: + return {"name": path, "type": "directory", "size": 0} + if out0[0]["size"] is None: + # if this is a whole remote file, update size using remote FS + prot, _ = split_protocol(self.references[path][0]) + out0[0]["size"] = self.fss[prot].size(self.references[path][0]) + return out0[0] + + async def _info(self, path, **kwargs): # calls fast sync code + return self.info(path) + + async def _rm_file(self, path, **kwargs): + self.references.pop( + path, None + ) # ignores FileNotFound, just as well for directories + self.dircache.clear() # this is a bit heavy handed + + async def _pipe_file(self, path, data, mode="overwrite", **kwargs): + if mode == "create" and self.exists(path): + raise FileExistsError + # can be str or bytes + self.references[path] = data + self.dircache.clear() # this is a bit heavy handed + + async def _put_file(self, lpath, rpath, mode="overwrite", **kwargs): + # puts binary + if mode == "create" and self.exists(rpath): + raise FileExistsError + with open(lpath, "rb") as f: + self.references[rpath] = f.read() + self.dircache.clear() # this is a bit heavy handed + + def save_json(self, url, **storage_options): + """Write modified references into new location""" + out = {} + for k, v in self.references.items(): + if isinstance(v, bytes): + try: + out[k] = v.decode("ascii") + except UnicodeDecodeError: + out[k] = (b"base64:" + base64.b64encode(v)).decode() + else: + out[k] = v + with fsspec.open(url, "wb", **storage_options) as f: + f.write(json.dumps({"version": 1, "refs": out}).encode()) + + +class ReferenceFile(AbstractBufferedFile): + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + size=None, + **kwargs, + ): + super().__init__( + fs, + path, + mode=mode, + block_size=block_size, + autocommit=autocommit, + size=size, + cache_type=cache_type, + cache_options=cache_options, + **kwargs, + ) + part_or_url, self.start, self.end = self.fs._cat_common(self.path) + protocol, _ = split_protocol(part_or_url) + self.src_fs = self.fs.fss[protocol] + self.src_path = part_or_url + self._f = None + + @property + def f(self): + if self._f is None or self._f.closed: + self._f = self.src_fs._open( + self.src_path, + mode=self.mode, + block_size=self.blocksize, + autocommit=self.autocommit, + cache_type="none", + **self.kwargs, + ) + return self._f + + def close(self): + if self._f is not None: + self._f.close() + return super().close() + + def _fetch_range(self, start, end): + start = start + self.start + end = min(end + self.start, self.end) + self.f.seek(start) + return self.f.read(end - start) diff --git a/lib/python3.12/site-packages/fsspec/implementations/sftp.py b/lib/python3.12/site-packages/fsspec/implementations/sftp.py new file mode 100644 index 0000000000000000000000000000000000000000..77f7b370cd246f9a9bfd34141afc3edd728d13c3 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/sftp.py @@ -0,0 +1,180 @@ +import datetime +import logging +import os +import types +import uuid +from stat import S_ISDIR, S_ISLNK + +import paramiko + +from .. import AbstractFileSystem +from ..utils import infer_storage_options + +logger = logging.getLogger("fsspec.sftp") + + +class SFTPFileSystem(AbstractFileSystem): + """Files over SFTP/SSH + + Peer-to-peer filesystem over SSH using paramiko. + + Note: if using this with the ``open`` or ``open_files``, with full URLs, + there is no way to tell if a path is relative, so all paths are assumed + to be absolute. + """ + + protocol = "sftp", "ssh" + + def __init__(self, host, **ssh_kwargs): + """ + + Parameters + ---------- + host: str + Hostname or IP as a string + temppath: str + Location on the server to put files, when within a transaction + ssh_kwargs: dict + Parameters passed on to connection. See details in + https://docs.paramiko.org/en/3.3/api/client.html#paramiko.client.SSHClient.connect + May include port, username, password... + """ + if self._cached: + return + super().__init__(**ssh_kwargs) + self.temppath = ssh_kwargs.pop("temppath", "/tmp") # remote temp directory + self.host = host + self.ssh_kwargs = ssh_kwargs + self._connect() + + def _connect(self): + logger.debug("Connecting to SFTP server %s", self.host) + self.client = paramiko.SSHClient() + self.client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + self.client.connect(self.host, **self.ssh_kwargs) + self.ftp = self.client.open_sftp() + + @classmethod + def _strip_protocol(cls, path): + return infer_storage_options(path)["path"] + + @staticmethod + def _get_kwargs_from_urls(urlpath): + out = infer_storage_options(urlpath) + out.pop("path", None) + out.pop("protocol", None) + return out + + def mkdir(self, path, create_parents=True, mode=511): + logger.debug("Creating folder %s", path) + if self.exists(path): + raise FileExistsError(f"File exists: {path}") + + if create_parents: + self.makedirs(path) + else: + self.ftp.mkdir(path, mode) + + def makedirs(self, path, exist_ok=False, mode=511): + if self.exists(path) and not exist_ok: + raise FileExistsError(f"File exists: {path}") + + parts = path.split("/") + new_path = "/" if path[:1] == "/" else "" + + for part in parts: + if part: + new_path = f"{new_path}/{part}" if new_path else part + if not self.exists(new_path): + self.ftp.mkdir(new_path, mode) + + def rmdir(self, path): + logger.debug("Removing folder %s", path) + self.ftp.rmdir(path) + + def info(self, path): + stat = self._decode_stat(self.ftp.stat(path)) + stat["name"] = path + return stat + + @staticmethod + def _decode_stat(stat, parent_path=None): + if S_ISDIR(stat.st_mode): + t = "directory" + elif S_ISLNK(stat.st_mode): + t = "link" + else: + t = "file" + out = { + "name": "", + "size": stat.st_size, + "type": t, + "uid": stat.st_uid, + "gid": stat.st_gid, + "time": datetime.datetime.fromtimestamp( + stat.st_atime, tz=datetime.timezone.utc + ), + "mtime": datetime.datetime.fromtimestamp( + stat.st_mtime, tz=datetime.timezone.utc + ), + } + if parent_path: + out["name"] = "/".join([parent_path.rstrip("/"), stat.filename]) + return out + + def ls(self, path, detail=False): + logger.debug("Listing folder %s", path) + stats = [self._decode_stat(stat, path) for stat in self.ftp.listdir_iter(path)] + if detail: + return stats + else: + paths = [stat["name"] for stat in stats] + return sorted(paths) + + def put(self, lpath, rpath, callback=None, **kwargs): + logger.debug("Put file %s into %s", lpath, rpath) + self.ftp.put(lpath, rpath) + + def get_file(self, rpath, lpath, **kwargs): + if self.isdir(rpath): + os.makedirs(lpath, exist_ok=True) + else: + self.ftp.get(self._strip_protocol(rpath), lpath) + + def _open(self, path, mode="rb", block_size=None, **kwargs): + """ + block_size: int or None + If 0, no buffering, if 1, line buffering, if >1, buffer that many + bytes, if None use default from paramiko. + """ + logger.debug("Opening file %s", path) + if kwargs.get("autocommit", True) is False: + # writes to temporary file, move on commit + path2 = "/".join([self.temppath, str(uuid.uuid4())]) + f = self.ftp.open(path2, mode, bufsize=block_size if block_size else -1) + f.temppath = path2 + f.targetpath = path + f.fs = self + f.commit = types.MethodType(commit_a_file, f) + f.discard = types.MethodType(discard_a_file, f) + else: + f = self.ftp.open(path, mode, bufsize=block_size if block_size else -1) + return f + + def _rm(self, path): + if self.isdir(path): + self.ftp.rmdir(path) + else: + self.ftp.remove(path) + + def mv(self, old, new): + logger.debug("Renaming %s into %s", old, new) + self.ftp.posix_rename(old, new) + + +def commit_a_file(self): + self.fs.mv(self.temppath, self.targetpath) + + +def discard_a_file(self): + self.fs._rm(self.temppath) diff --git a/lib/python3.12/site-packages/fsspec/implementations/smb.py b/lib/python3.12/site-packages/fsspec/implementations/smb.py new file mode 100644 index 0000000000000000000000000000000000000000..db6b3f5c3702de90cf121ccca49f3ca2b580df9f --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/smb.py @@ -0,0 +1,416 @@ +""" +This module contains SMBFileSystem class responsible for handling access to +Windows Samba network shares by using package smbprotocol +""" + +import datetime +import re +import uuid +from stat import S_ISDIR, S_ISLNK + +import smbclient +import smbprotocol.exceptions + +from .. import AbstractFileSystem +from ..utils import infer_storage_options + +# ! pylint: disable=bad-continuation + + +class SMBFileSystem(AbstractFileSystem): + """Allow reading and writing to Windows and Samba network shares. + + When using `fsspec.open()` for getting a file-like object the URI + should be specified as this format: + ``smb://workgroup;user:password@server:port/share/folder/file.csv``. + + Example:: + + >>> import fsspec + >>> with fsspec.open( + ... 'smb://myuser:mypassword@myserver.com/' 'share/folder/file.csv' + ... ) as smbfile: + ... df = pd.read_csv(smbfile, sep='|', header=None) + + Note that you need to pass in a valid hostname or IP address for the host + component of the URL. Do not use the Windows/NetBIOS machine name for the + host component. + + The first component of the path in the URL points to the name of the shared + folder. Subsequent path components will point to the directory/folder/file. + + The URL components ``workgroup`` , ``user``, ``password`` and ``port`` may be + optional. + + .. note:: + + For working this source require `smbprotocol`_ to be installed, e.g.:: + + $ pip install smbprotocol + # or + # pip install smbprotocol[kerberos] + + .. _smbprotocol: https://github.com/jborean93/smbprotocol#requirements + + Note: if using this with the ``open`` or ``open_files``, with full URLs, + there is no way to tell if a path is relative, so all paths are assumed + to be absolute. + """ + + protocol = "smb" + + # pylint: disable=too-many-arguments + def __init__( + self, + host, + port=None, + username=None, + password=None, + timeout=60, + encrypt=None, + share_access=None, + register_session_retries=4, + register_session_retry_wait=1, + register_session_retry_factor=10, + auto_mkdir=False, + **kwargs, + ): + """ + You can use _get_kwargs_from_urls to get some kwargs from + a reasonable SMB url. + + Authentication will be anonymous or integrated if username/password are not + given. + + Parameters + ---------- + host: str + The remote server name/ip to connect to + port: int or None + Port to connect with. Usually 445, sometimes 139. + username: str or None + Username to connect with. Required if Kerberos auth is not being used. + password: str or None + User's password on the server, if using username + timeout: int + Connection timeout in seconds + encrypt: bool + Whether to force encryption or not, once this has been set to True + the session cannot be changed back to False. + share_access: str or None + Specifies the default access applied to file open operations + performed with this file system object. + This affects whether other processes can concurrently open a handle + to the same file. + + - None (the default): exclusively locks the file until closed. + - 'r': Allow other handles to be opened with read access. + - 'w': Allow other handles to be opened with write access. + - 'd': Allow other handles to be opened with delete access. + register_session_retries: int + Number of retries to register a session with the server. Retries are not performed + for authentication errors, as they are considered as invalid credentials and not network + issues. If set to negative value, no register attempts will be performed. + register_session_retry_wait: int + Time in seconds to wait between each retry. Number must be non-negative. + register_session_retry_factor: int + Base factor for the wait time between each retry. The wait time + is calculated using exponential function. For factor=1 all wait times + will be equal to `register_session_retry_wait`. For any number of retries, + the last wait time will be equal to `register_session_retry_wait` and for retries>1 + the first wait time will be equal to `register_session_retry_wait / factor`. + Number must be equal to or greater than 1. Optimal factor is 10. + auto_mkdir: bool + Whether, when opening a file, the directory containing it should + be created (if it doesn't already exist). This is assumed by pyarrow + and zarr-python code. + """ + super().__init__(**kwargs) + self.host = host + self.port = port + self.username = username + self.password = password + self.timeout = timeout + self.encrypt = encrypt + self.temppath = kwargs.pop("temppath", "") + self.share_access = share_access + self.register_session_retries = register_session_retries + if register_session_retry_wait < 0: + raise ValueError( + "register_session_retry_wait must be a non-negative integer" + ) + self.register_session_retry_wait = register_session_retry_wait + if register_session_retry_factor < 1: + raise ValueError( + "register_session_retry_factor must be a positive " + "integer equal to or greater than 1" + ) + self.register_session_retry_factor = register_session_retry_factor + self.auto_mkdir = auto_mkdir + self._connect() + + @property + def _port(self): + return 445 if self.port is None else self.port + + def _connect(self): + import time + + if self.register_session_retries <= -1: + return + + retried_errors = [] + + wait_time = self.register_session_retry_wait + n_waits = ( + self.register_session_retries - 1 + ) # -1 = No wait time after the last retry + factor = self.register_session_retry_factor + + # Generate wait times for each retry attempt. + # Wait times are calculated using exponential function. For factor=1 all wait times + # will be equal to `wait`. For any number of retries the last wait time will be + # equal to `wait` and for retries>2 the first wait time will be equal to `wait / factor`. + wait_times = iter( + factor ** (n / n_waits - 1) * wait_time for n in range(0, n_waits + 1) + ) + + for attempt in range(self.register_session_retries + 1): + try: + smbclient.register_session( + self.host, + username=self.username, + password=self.password, + port=self._port, + encrypt=self.encrypt, + connection_timeout=self.timeout, + ) + return + except ( + smbprotocol.exceptions.SMBAuthenticationError, + smbprotocol.exceptions.LogonFailure, + ): + # These exceptions should not be repeated, as they clearly indicate + # that the credentials are invalid and not a network issue. + raise + except ValueError as exc: + if re.findall(r"\[Errno -\d+]", str(exc)): + # This exception is raised by the smbprotocol.transport:Tcp.connect + # and originates from socket.gaierror (OSError). These exceptions might + # be raised due to network instability. We will retry to connect. + retried_errors.append(exc) + else: + # All another ValueError exceptions should be raised, as they are not + # related to network issues. + raise + except Exception as exc: + # Save the exception and retry to connect. This except might be dropped + # in the future, once all exceptions suited for retry are identified. + retried_errors.append(exc) + + if attempt < self.register_session_retries: + time.sleep(next(wait_times)) + + # Raise last exception to inform user about the connection issues. + # Note: Should we use ExceptionGroup to raise all exceptions? + raise retried_errors[-1] + + @classmethod + def _strip_protocol(cls, path): + return infer_storage_options(path)["path"] + + @staticmethod + def _get_kwargs_from_urls(path): + # smb://workgroup;user:password@host:port/share/folder/file.csv + out = infer_storage_options(path) + out.pop("path", None) + out.pop("protocol", None) + return out + + def mkdir(self, path, create_parents=True, **kwargs): + wpath = _as_unc_path(self.host, path) + if create_parents: + smbclient.makedirs(wpath, exist_ok=False, port=self._port, **kwargs) + else: + smbclient.mkdir(wpath, port=self._port, **kwargs) + + def makedirs(self, path, exist_ok=False): + if _share_has_path(path): + wpath = _as_unc_path(self.host, path) + smbclient.makedirs(wpath, exist_ok=exist_ok, port=self._port) + + def rmdir(self, path): + if _share_has_path(path): + wpath = _as_unc_path(self.host, path) + smbclient.rmdir(wpath, port=self._port) + + def info(self, path, **kwargs): + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port, **kwargs) + if S_ISDIR(stats.st_mode): + stype = "directory" + elif S_ISLNK(stats.st_mode): + stype = "link" + else: + stype = "file" + res = { + "name": path + "/" if stype == "directory" else path, + "size": stats.st_size, + "type": stype, + "uid": stats.st_uid, + "gid": stats.st_gid, + "time": stats.st_atime, + "mtime": stats.st_mtime, + } + return res + + def created(self, path): + """Return the created timestamp of a file as a datetime.datetime""" + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port) + return datetime.datetime.fromtimestamp(stats.st_ctime, tz=datetime.timezone.utc) + + def modified(self, path): + """Return the modified timestamp of a file as a datetime.datetime""" + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port) + return datetime.datetime.fromtimestamp(stats.st_mtime, tz=datetime.timezone.utc) + + def ls(self, path, detail=True, **kwargs): + unc = _as_unc_path(self.host, path) + listed = smbclient.listdir(unc, port=self._port, **kwargs) + dirs = ["/".join([path.rstrip("/"), p]) for p in listed] + if detail: + dirs = [self.info(d) for d in dirs] + return dirs + + # pylint: disable=too-many-arguments + def _open( + self, + path, + mode="rb", + block_size=-1, + autocommit=True, + cache_options=None, + **kwargs, + ): + """ + block_size: int or None + If 0, no buffering, 1, line buffering, >1, buffer that many bytes + + Notes + ----- + By specifying 'share_access' in 'kwargs' it is possible to override the + default shared access setting applied in the constructor of this object. + """ + if self.auto_mkdir and "w" in mode: + self.makedirs(self._parent(path), exist_ok=True) + bls = block_size if block_size is not None and block_size >= 0 else -1 + wpath = _as_unc_path(self.host, path) + share_access = kwargs.pop("share_access", self.share_access) + if "w" in mode and autocommit is False: + temp = _as_temp_path(self.host, path, self.temppath) + return SMBFileOpener( + wpath, temp, mode, port=self._port, block_size=bls, **kwargs + ) + return smbclient.open_file( + wpath, + mode, + buffering=bls, + share_access=share_access, + port=self._port, + **kwargs, + ) + + def copy(self, path1, path2, **kwargs): + """Copy within two locations in the same filesystem""" + wpath1 = _as_unc_path(self.host, path1) + wpath2 = _as_unc_path(self.host, path2) + if self.auto_mkdir: + self.makedirs(self._parent(path2), exist_ok=True) + smbclient.copyfile(wpath1, wpath2, port=self._port, **kwargs) + + def _rm(self, path): + if _share_has_path(path): + wpath = _as_unc_path(self.host, path) + stats = smbclient.stat(wpath, port=self._port) + if S_ISDIR(stats.st_mode): + smbclient.rmdir(wpath, port=self._port) + else: + smbclient.remove(wpath, port=self._port) + + def mv(self, path1, path2, recursive=None, maxdepth=None, **kwargs): + wpath1 = _as_unc_path(self.host, path1) + wpath2 = _as_unc_path(self.host, path2) + smbclient.rename(wpath1, wpath2, port=self._port, **kwargs) + + +def _as_unc_path(host, path): + rpath = path.replace("/", "\\") + unc = f"\\\\{host}{rpath}" + return unc + + +def _as_temp_path(host, path, temppath): + share = path.split("/")[1] + temp_file = f"/{share}{temppath}/{uuid.uuid4()}" + unc = _as_unc_path(host, temp_file) + return unc + + +def _share_has_path(path): + parts = path.count("/") + if path.endswith("/"): + return parts > 2 + return parts > 1 + + +class SMBFileOpener: + """writes to remote temporary file, move on commit""" + + def __init__(self, path, temp, mode, port=445, block_size=-1, **kwargs): + self.path = path + self.temp = temp + self.mode = mode + self.block_size = block_size + self.kwargs = kwargs + self.smbfile = None + self._incontext = False + self.port = port + self._open() + + def _open(self): + if self.smbfile is None or self.smbfile.closed: + self.smbfile = smbclient.open_file( + self.temp, + self.mode, + port=self.port, + buffering=self.block_size, + **self.kwargs, + ) + + def commit(self): + """Move temp file to definitive on success.""" + # TODO: use transaction support in SMB protocol + smbclient.replace(self.temp, self.path, port=self.port) + + def discard(self): + """Remove the temp file on failure.""" + smbclient.remove(self.temp, port=self.port) + + def __fspath__(self): + return self.path + + def __iter__(self): + return self.smbfile.__iter__() + + def __getattr__(self, item): + return getattr(self.smbfile, item) + + def __enter__(self): + self._incontext = True + return self.smbfile.__enter__() + + def __exit__(self, exc_type, exc_value, traceback): + self._incontext = False + self.smbfile.__exit__(exc_type, exc_value, traceback) diff --git a/lib/python3.12/site-packages/fsspec/implementations/tar.py b/lib/python3.12/site-packages/fsspec/implementations/tar.py new file mode 100644 index 0000000000000000000000000000000000000000..412e5ba4d2cdea7db090dc96412e697909a38d78 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/tar.py @@ -0,0 +1,124 @@ +import logging +import tarfile + +import fsspec +from fsspec.archive import AbstractArchiveFileSystem +from fsspec.compression import compr +from fsspec.utils import infer_compression + +typemap = {b"0": "file", b"5": "directory"} + +logger = logging.getLogger("tar") + + +class TarFileSystem(AbstractArchiveFileSystem): + """Compressed Tar archives as a file-system (read-only) + + Supports the following formats: + tar.gz, tar.bz2, tar.xz + """ + + root_marker = "" + protocol = "tar" + cachable = False + + def __init__( + self, + fo="", + index_store=None, + target_options=None, + target_protocol=None, + compression=None, + **kwargs, + ): + super().__init__(**kwargs) + target_options = target_options or {} + + if isinstance(fo, str): + self.of = fsspec.open(fo, protocol=target_protocol, **target_options) + fo = self.of.open() # keep the reference + + # Try to infer compression. + if compression is None: + name = None + + # Try different ways to get hold of the filename. `fo` might either + # be a `fsspec.LocalFileOpener`, an `io.BufferedReader` or an + # `fsspec.AbstractFileSystem` instance. + try: + # Amended io.BufferedReader or similar. + # This uses a "protocol extension" where original filenames are + # propagated to archive-like filesystems in order to let them + # infer the right compression appropriately. + if hasattr(fo, "original"): + name = fo.original + + # fsspec.LocalFileOpener + elif hasattr(fo, "path"): + name = fo.path + + # io.BufferedReader + elif hasattr(fo, "name"): + name = fo.name + + # fsspec.AbstractFileSystem + elif hasattr(fo, "info"): + name = fo.info()["name"] + + except Exception as ex: + logger.warning( + f"Unable to determine file name, not inferring compression: {ex}" + ) + + if name is not None: + compression = infer_compression(name) + logger.info(f"Inferred compression {compression} from file name {name}") + + if compression is not None: + # TODO: tarfile already implements compression with modes like "'r:gz'", + # but then would seek to offset in the file work? + fo = compr[compression](fo) + + self._fo_ref = fo + self.fo = fo # the whole instance is a context + self.tar = tarfile.TarFile(fileobj=self.fo) + self.dir_cache = None + + self.index_store = index_store + self.index = None + self._index() + + def _index(self): + # TODO: load and set saved index, if exists + out = {} + for ti in self.tar: + info = ti.get_info() + info["type"] = typemap.get(info["type"], "file") + name = ti.get_info()["name"].rstrip("/") + out[name] = (info, ti.offset_data) + + self.index = out + # TODO: save index to self.index_store here, if set + + def _get_dirs(self): + if self.dir_cache is not None: + return + + # This enables ls to get directories as children as well as files + self.dir_cache = { + dirname: {"name": dirname, "size": 0, "type": "directory"} + for dirname in self._all_dirnames(self.tar.getnames()) + } + for member in self.tar.getmembers(): + info = member.get_info() + info["name"] = info["name"].rstrip("/") + info["type"] = typemap.get(info["type"], "file") + self.dir_cache[info["name"]] = info + + def _open(self, path, mode="rb", **kwargs): + if mode != "rb": + raise ValueError("Read-only filesystem implementation") + details, offset = self.index[path] + if details["type"] != "file": + raise ValueError("Can only handle regular files") + return self.tar.extractfile(path) diff --git a/lib/python3.12/site-packages/fsspec/implementations/webhdfs.py b/lib/python3.12/site-packages/fsspec/implementations/webhdfs.py new file mode 100644 index 0000000000000000000000000000000000000000..c6e0d8446f875ec9578cccf055aeb47cd1c2e996 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/webhdfs.py @@ -0,0 +1,485 @@ +# https://hadoop.apache.org/docs/r1.0.4/webhdfs.html + +import logging +import os +import secrets +import shutil +import tempfile +import uuid +from contextlib import suppress +from urllib.parse import quote + +import requests + +from ..spec import AbstractBufferedFile, AbstractFileSystem +from ..utils import infer_storage_options, tokenize + +logger = logging.getLogger("webhdfs") + + +class WebHDFS(AbstractFileSystem): + """ + Interface to HDFS over HTTP using the WebHDFS API. Supports also HttpFS gateways. + + Four auth mechanisms are supported: + + insecure: no auth is done, and the user is assumed to be whoever they + say they are (parameter ``user``), or a predefined value such as + "dr.who" if not given + spnego: when kerberos authentication is enabled, auth is negotiated by + requests_kerberos https://github.com/requests/requests-kerberos . + This establishes a session based on existing kinit login and/or + specified principal/password; parameters are passed with ``kerb_kwargs`` + token: uses an existing Hadoop delegation token from another secured + service. Indeed, this client can also generate such tokens when + not insecure. Note that tokens expire, but can be renewed (by a + previously specified user) and may allow for proxying. + basic-auth: used when both parameter ``user`` and parameter ``password`` + are provided. + + """ + + tempdir = str(tempfile.gettempdir()) + protocol = "webhdfs", "webHDFS" + + def __init__( + self, + host, + port=50070, + kerberos=False, + token=None, + user=None, + password=None, + proxy_to=None, + kerb_kwargs=None, + data_proxy=None, + use_https=False, + session_cert=None, + session_verify=True, + **kwargs, + ): + """ + Parameters + ---------- + host: str + Name-node address + port: int + Port for webHDFS + kerberos: bool + Whether to authenticate with kerberos for this connection + token: str or None + If given, use this token on every call to authenticate. A user + and user-proxy may be encoded in the token and should not be also + given + user: str or None + If given, assert the user name to connect with + password: str or None + If given, assert the password to use for basic auth. If password + is provided, user must be provided also + proxy_to: str or None + If given, the user has the authority to proxy, and this value is + the user in who's name actions are taken + kerb_kwargs: dict + Any extra arguments for HTTPKerberosAuth, see + ``_ + data_proxy: dict, callable or None + If given, map data-node addresses. This can be necessary if the + HDFS cluster is behind a proxy, running on Docker or otherwise has + a mismatch between the host-names given by the name-node and the + address by which to refer to them from the client. If a dict, + maps host names ``host->data_proxy[host]``; if a callable, full + URLs are passed, and function must conform to + ``url->data_proxy(url)``. + use_https: bool + Whether to connect to the Name-node using HTTPS instead of HTTP + session_cert: str or Tuple[str, str] or None + Path to a certificate file, or tuple of (cert, key) files to use + for the requests.Session + session_verify: str, bool or None + Path to a certificate file to use for verifying the requests.Session. + kwargs + """ + if self._cached: + return + super().__init__(**kwargs) + self.url = f"{'https' if use_https else 'http'}://{host}:{port}/webhdfs/v1" + self.kerb = kerberos + self.kerb_kwargs = kerb_kwargs or {} + self.pars = {} + self.proxy = data_proxy or {} + if token is not None: + if user is not None or proxy_to is not None: + raise ValueError( + "If passing a delegation token, must not set " + "user or proxy_to, as these are encoded in the" + " token" + ) + self.pars["delegation"] = token + self.user = user + self.password = password + + if password is not None: + if user is None: + raise ValueError( + "If passing a password, the user must also be" + "set in order to set up the basic-auth" + ) + else: + if user is not None: + self.pars["user.name"] = user + + if proxy_to is not None: + self.pars["doas"] = proxy_to + if kerberos and user is not None: + raise ValueError( + "If using Kerberos auth, do not specify the " + "user, this is handled by kinit." + ) + + self.session_cert = session_cert + self.session_verify = session_verify + + self._connect() + + self._fsid = f"webhdfs_{tokenize(host, port)}" + + @property + def fsid(self): + return self._fsid + + def _connect(self): + self.session = requests.Session() + + if self.session_cert: + self.session.cert = self.session_cert + + self.session.verify = self.session_verify + + if self.kerb: + from requests_kerberos import HTTPKerberosAuth + + self.session.auth = HTTPKerberosAuth(**self.kerb_kwargs) + + if self.user is not None and self.password is not None: + from requests.auth import HTTPBasicAuth + + self.session.auth = HTTPBasicAuth(self.user, self.password) + + def _call(self, op, method="get", path=None, data=None, redirect=True, **kwargs): + path = self._strip_protocol(path) if path is not None else "" + url = self._apply_proxy(self.url + quote(path, safe="/=")) + args = kwargs.copy() + args.update(self.pars) + args["op"] = op.upper() + logger.debug("sending %s with %s", url, method) + out = self.session.request( + method=method.upper(), + url=url, + params=args, + data=data, + allow_redirects=redirect, + ) + if out.status_code in [400, 401, 403, 404, 500]: + try: + err = out.json() + msg = err["RemoteException"]["message"] + exp = err["RemoteException"]["exception"] + except (ValueError, KeyError): + pass + else: + if exp in ["IllegalArgumentException", "UnsupportedOperationException"]: + raise ValueError(msg) + elif exp in ["SecurityException", "AccessControlException"]: + raise PermissionError(msg) + elif exp in ["FileNotFoundException"]: + raise FileNotFoundError(msg) + else: + raise RuntimeError(msg) + out.raise_for_status() + return out + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + replication=None, + permissions=None, + **kwargs, + ): + """ + + Parameters + ---------- + path: str + File location + mode: str + 'rb', 'wb', etc. + block_size: int + Client buffer size for read-ahead or write buffer + autocommit: bool + If False, writes to temporary file that only gets put in final + location upon commit + replication: int + Number of copies of file on the cluster, write mode only + permissions: str or int + posix permissions, write mode only + kwargs + + Returns + ------- + WebHDFile instance + """ + block_size = block_size or self.blocksize + return WebHDFile( + self, + path, + mode=mode, + block_size=block_size, + tempdir=self.tempdir, + autocommit=autocommit, + replication=replication, + permissions=permissions, + ) + + @staticmethod + def _process_info(info): + info["type"] = info["type"].lower() + info["size"] = info["length"] + return info + + @classmethod + def _strip_protocol(cls, path): + return infer_storage_options(path)["path"] + + @staticmethod + def _get_kwargs_from_urls(urlpath): + out = infer_storage_options(urlpath) + out.pop("path", None) + out.pop("protocol", None) + if "username" in out: + out["user"] = out.pop("username") + return out + + def info(self, path): + out = self._call("GETFILESTATUS", path=path) + info = out.json()["FileStatus"] + info["name"] = path + return self._process_info(info) + + def ls(self, path, detail=False): + out = self._call("LISTSTATUS", path=path) + infos = out.json()["FileStatuses"]["FileStatus"] + for info in infos: + self._process_info(info) + info["name"] = path.rstrip("/") + "/" + info["pathSuffix"] + if detail: + return sorted(infos, key=lambda i: i["name"]) + else: + return sorted(info["name"] for info in infos) + + def content_summary(self, path): + """Total numbers of files, directories and bytes under path""" + out = self._call("GETCONTENTSUMMARY", path=path) + return out.json()["ContentSummary"] + + def ukey(self, path): + """Checksum info of file, giving method and result""" + out = self._call("GETFILECHECKSUM", path=path, redirect=False) + if "Location" in out.headers: + location = self._apply_proxy(out.headers["Location"]) + out2 = self.session.get(location) + out2.raise_for_status() + return out2.json()["FileChecksum"] + else: + out.raise_for_status() + return out.json()["FileChecksum"] + + def home_directory(self): + """Get user's home directory""" + out = self._call("GETHOMEDIRECTORY") + return out.json()["Path"] + + def get_delegation_token(self, renewer=None): + """Retrieve token which can give the same authority to other uses + + Parameters + ---------- + renewer: str or None + User who may use this token; if None, will be current user + """ + if renewer: + out = self._call("GETDELEGATIONTOKEN", renewer=renewer) + else: + out = self._call("GETDELEGATIONTOKEN") + t = out.json()["Token"] + if t is None: + raise ValueError("No token available for this user/security context") + return t["urlString"] + + def renew_delegation_token(self, token): + """Make token live longer. Returns new expiry time""" + out = self._call("RENEWDELEGATIONTOKEN", method="put", token=token) + return out.json()["long"] + + def cancel_delegation_token(self, token): + """Stop the token from being useful""" + self._call("CANCELDELEGATIONTOKEN", method="put", token=token) + + def chmod(self, path, mod): + """Set the permission at path + + Parameters + ---------- + path: str + location to set (file or directory) + mod: str or int + posix epresentation or permission, give as oct string, e.g, '777' + or 0o777 + """ + self._call("SETPERMISSION", method="put", path=path, permission=mod) + + def chown(self, path, owner=None, group=None): + """Change owning user and/or group""" + kwargs = {} + if owner is not None: + kwargs["owner"] = owner + if group is not None: + kwargs["group"] = group + self._call("SETOWNER", method="put", path=path, **kwargs) + + def set_replication(self, path, replication): + """ + Set file replication factor + + Parameters + ---------- + path: str + File location (not for directories) + replication: int + Number of copies of file on the cluster. Should be smaller than + number of data nodes; normally 3 on most systems. + """ + self._call("SETREPLICATION", path=path, method="put", replication=replication) + + def mkdir(self, path, **kwargs): + self._call("MKDIRS", method="put", path=path) + + def makedirs(self, path, exist_ok=False): + if exist_ok is False and self.exists(path): + raise FileExistsError(path) + self.mkdir(path) + + def mv(self, path1, path2, **kwargs): + self._call("RENAME", method="put", path=path1, destination=path2) + + def rm(self, path, recursive=False, **kwargs): + self._call( + "DELETE", + method="delete", + path=path, + recursive="true" if recursive else "false", + ) + + def rm_file(self, path, **kwargs): + self.rm(path) + + def cp_file(self, lpath, rpath, **kwargs): + with self.open(lpath) as lstream: + tmp_fname = "/".join([self._parent(rpath), f".tmp.{secrets.token_hex(16)}"]) + # Perform an atomic copy (stream to a temporary file and + # move it to the actual destination). + try: + with self.open(tmp_fname, "wb") as rstream: + shutil.copyfileobj(lstream, rstream) + self.mv(tmp_fname, rpath) + except BaseException: + with suppress(FileNotFoundError): + self.rm(tmp_fname) + raise + + def _apply_proxy(self, location): + if self.proxy and callable(self.proxy): + location = self.proxy(location) + elif self.proxy: + # as a dict + for k, v in self.proxy.items(): + location = location.replace(k, v, 1) + return location + + +class WebHDFile(AbstractBufferedFile): + """A file living in HDFS over webHDFS""" + + def __init__(self, fs, path, **kwargs): + super().__init__(fs, path, **kwargs) + kwargs = kwargs.copy() + if kwargs.get("permissions", None) is None: + kwargs.pop("permissions", None) + if kwargs.get("replication", None) is None: + kwargs.pop("replication", None) + self.permissions = kwargs.pop("permissions", 511) + tempdir = kwargs.pop("tempdir") + if kwargs.pop("autocommit", False) is False: + self.target = self.path + self.path = os.path.join(tempdir, str(uuid.uuid4())) + + def _upload_chunk(self, final=False): + """Write one part of a multi-block file upload + + Parameters + ========== + final: bool + This is the last block, so should complete file, if + self.autocommit is True. + """ + out = self.fs.session.post( + self.location, + data=self.buffer.getvalue(), + headers={"content-type": "application/octet-stream"}, + ) + out.raise_for_status() + return True + + def _initiate_upload(self): + """Create remote file/upload""" + kwargs = self.kwargs.copy() + if "a" in self.mode: + op, method = "APPEND", "POST" + else: + op, method = "CREATE", "PUT" + kwargs["overwrite"] = "true" + out = self.fs._call(op, method, self.path, redirect=False, **kwargs) + location = self.fs._apply_proxy(out.headers["Location"]) + if "w" in self.mode: + # create empty file to append to + out2 = self.fs.session.put( + location, headers={"content-type": "application/octet-stream"} + ) + out2.raise_for_status() + # after creating empty file, change location to append to + out2 = self.fs._call("APPEND", "POST", self.path, redirect=False, **kwargs) + self.location = self.fs._apply_proxy(out2.headers["Location"]) + + def _fetch_range(self, start, end): + start = max(start, 0) + end = min(self.size, end) + if start >= end or start >= self.size: + return b"" + out = self.fs._call( + "OPEN", path=self.path, offset=start, length=end - start, redirect=False + ) + out.raise_for_status() + if "Location" in out.headers: + location = out.headers["Location"] + out2 = self.fs.session.get(self.fs._apply_proxy(location)) + return out2.content + else: + return out.content + + def commit(self): + self.fs.mv(self.path, self.target) + + def discard(self): + self.fs.rm(self.path) diff --git a/lib/python3.12/site-packages/fsspec/implementations/zip.py b/lib/python3.12/site-packages/fsspec/implementations/zip.py new file mode 100644 index 0000000000000000000000000000000000000000..6db3ae27806106a19a366886ab4b183f85c1cb1a --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/implementations/zip.py @@ -0,0 +1,177 @@ +import os +import zipfile + +import fsspec +from fsspec.archive import AbstractArchiveFileSystem + + +class ZipFileSystem(AbstractArchiveFileSystem): + """Read/Write contents of ZIP archive as a file-system + + Keeps file object open while instance lives. + + This class is pickleable, but not necessarily thread-safe + """ + + root_marker = "" + protocol = "zip" + cachable = False + + def __init__( + self, + fo="", + mode="r", + target_protocol=None, + target_options=None, + compression=zipfile.ZIP_STORED, + allowZip64=True, + compresslevel=None, + **kwargs, + ): + """ + Parameters + ---------- + fo: str or file-like + Contains ZIP, and must exist. If a str, will fetch file using + :meth:`~fsspec.open_files`, which must return one file exactly. + mode: str + Accept: "r", "w", "a" + target_protocol: str (optional) + If ``fo`` is a string, this value can be used to override the + FS protocol inferred from a URL + target_options: dict (optional) + Kwargs passed when instantiating the target FS, if ``fo`` is + a string. + compression, allowZip64, compresslevel: passed to ZipFile + Only relevant when creating a ZIP + """ + super().__init__(self, **kwargs) + if mode not in set("rwa"): + raise ValueError(f"mode '{mode}' no understood") + self.mode = mode + if isinstance(fo, (str, os.PathLike)): + if mode == "a": + m = "r+b" + else: + m = mode + "b" + fo = fsspec.open( + fo, mode=m, protocol=target_protocol, **(target_options or {}) + ) + self.force_zip_64 = allowZip64 + self.of = fo + self.fo = fo.__enter__() # the whole instance is a context + self.zip = zipfile.ZipFile( + self.fo, + mode=mode, + compression=compression, + allowZip64=allowZip64, + compresslevel=compresslevel, + ) + self.dir_cache = None + + @classmethod + def _strip_protocol(cls, path): + # zip file paths are always relative to the archive root + return super()._strip_protocol(path).lstrip("/") + + def __del__(self): + if hasattr(self, "zip"): + self.close() + del self.zip + + def close(self): + """Commits any write changes to the file. Done on ``del`` too.""" + self.zip.close() + + def _get_dirs(self): + if self.dir_cache is None or self.mode in set("wa"): + # when writing, dir_cache is always in the ZipFile's attributes, + # not read from the file. + files = self.zip.infolist() + self.dir_cache = { + dirname.rstrip("/"): { + "name": dirname.rstrip("/"), + "size": 0, + "type": "directory", + } + for dirname in self._all_dirnames(self.zip.namelist()) + } + for z in files: + f = {s: getattr(z, s, None) for s in zipfile.ZipInfo.__slots__} + f.update( + { + "name": z.filename.rstrip("/"), + "size": z.file_size, + "type": ("directory" if z.is_dir() else "file"), + } + ) + self.dir_cache[f["name"]] = f + + def pipe_file(self, path, value, **kwargs): + # override upstream, because we know the exact file size in this case + self.zip.writestr(path, value, **kwargs) + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + path = self._strip_protocol(path) + if "r" in mode and self.mode in set("wa"): + if self.exists(path): + raise OSError("ZipFS can only be open for reading or writing, not both") + raise FileNotFoundError(path) + if "r" in self.mode and "w" in mode: + raise OSError("ZipFS can only be open for reading or writing, not both") + out = self.zip.open(path, mode.strip("b"), force_zip64=self.force_zip_64) + if "r" in mode: + info = self.info(path) + out.size = info["size"] + out.name = info["name"] + return out + + def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + # Remove the leading slash, as the zip file paths are always + # given without a leading slash + path = path.lstrip("/") + path_parts = list(filter(lambda s: bool(s), path.split("/"))) + + def _matching_starts(file_path): + file_parts = filter(lambda s: bool(s), file_path.split("/")) + return all(a == b for a, b in zip(path_parts, file_parts)) + + self._get_dirs() + + result = {} + # To match posix find, if an exact file name is given, we should + # return only that file + if path in self.dir_cache and self.dir_cache[path]["type"] == "file": + result[path] = self.dir_cache[path] + return result if detail else [path] + + for file_path, file_info in self.dir_cache.items(): + if not (path == "" or _matching_starts(file_path)): + continue + + if file_info["type"] == "directory": + if withdirs: + if file_path not in result: + result[file_path.strip("/")] = file_info + continue + + if file_path not in result: + result[file_path] = file_info if detail else None + + if maxdepth: + path_depth = path.count("/") + result = { + k: v for k, v in result.items() if k.count("/") - path_depth < maxdepth + } + return result if detail else sorted(result) diff --git a/lib/python3.12/site-packages/fsspec/json.py b/lib/python3.12/site-packages/fsspec/json.py new file mode 100644 index 0000000000000000000000000000000000000000..69cead04509a1ebc9175ec5ad449c8a866b60444 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/json.py @@ -0,0 +1,121 @@ +import json +from contextlib import suppress +from pathlib import PurePath +from typing import ( + Any, + Callable, + ClassVar, + Dict, + List, + Mapping, + Optional, + Sequence, + Tuple, +) + +from .registry import _import_class, get_filesystem_class +from .spec import AbstractFileSystem + + +class FilesystemJSONEncoder(json.JSONEncoder): + include_password: ClassVar[bool] = True + + def default(self, o: Any) -> Any: + if isinstance(o, AbstractFileSystem): + return o.to_dict(include_password=self.include_password) + if isinstance(o, PurePath): + cls = type(o) + return {"cls": f"{cls.__module__}.{cls.__name__}", "str": str(o)} + + return super().default(o) + + def make_serializable(self, obj: Any) -> Any: + """ + Recursively converts an object so that it can be JSON serialized via + :func:`json.dumps` and :func:`json.dump`, without actually calling + said functions. + """ + if isinstance(obj, (str, int, float, bool)): + return obj + if isinstance(obj, Mapping): + return {k: self.make_serializable(v) for k, v in obj.items()} + if isinstance(obj, Sequence): + return [self.make_serializable(v) for v in obj] + + return self.default(obj) + + +class FilesystemJSONDecoder(json.JSONDecoder): + def __init__( + self, + *, + object_hook: Optional[Callable[[Dict[str, Any]], Any]] = None, + parse_float: Optional[Callable[[str], Any]] = None, + parse_int: Optional[Callable[[str], Any]] = None, + parse_constant: Optional[Callable[[str], Any]] = None, + strict: bool = True, + object_pairs_hook: Optional[Callable[[List[Tuple[str, Any]]], Any]] = None, + ) -> None: + self.original_object_hook = object_hook + + super().__init__( + object_hook=self.custom_object_hook, + parse_float=parse_float, + parse_int=parse_int, + parse_constant=parse_constant, + strict=strict, + object_pairs_hook=object_pairs_hook, + ) + + @classmethod + def try_resolve_path_cls(cls, dct: Dict[str, Any]): + with suppress(Exception): + fqp = dct["cls"] + + path_cls = _import_class(fqp) + + if issubclass(path_cls, PurePath): + return path_cls + + return None + + @classmethod + def try_resolve_fs_cls(cls, dct: Dict[str, Any]): + with suppress(Exception): + if "cls" in dct: + try: + fs_cls = _import_class(dct["cls"]) + if issubclass(fs_cls, AbstractFileSystem): + return fs_cls + except Exception: + if "protocol" in dct: # Fallback if cls cannot be imported + return get_filesystem_class(dct["protocol"]) + + raise + + return None + + def custom_object_hook(self, dct: Dict[str, Any]): + if "cls" in dct: + if (obj_cls := self.try_resolve_fs_cls(dct)) is not None: + return AbstractFileSystem.from_dict(dct) + if (obj_cls := self.try_resolve_path_cls(dct)) is not None: + return obj_cls(dct["str"]) + + if self.original_object_hook is not None: + return self.original_object_hook(dct) + + return dct + + def unmake_serializable(self, obj: Any) -> Any: + """ + Inverse function of :meth:`FilesystemJSONEncoder.make_serializable`. + """ + if isinstance(obj, dict): + obj = self.custom_object_hook(obj) + if isinstance(obj, dict): + return {k: self.unmake_serializable(v) for k, v in obj.items()} + if isinstance(obj, (list, tuple)): + return [self.unmake_serializable(v) for v in obj] + + return obj diff --git a/lib/python3.12/site-packages/fsspec/mapping.py b/lib/python3.12/site-packages/fsspec/mapping.py new file mode 100644 index 0000000000000000000000000000000000000000..752eef35273b13eded7297e2e801b58e436a25b1 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/mapping.py @@ -0,0 +1,251 @@ +import array +import logging +import posixpath +import warnings +from collections.abc import MutableMapping +from functools import cached_property + +from fsspec.core import url_to_fs + +logger = logging.getLogger("fsspec.mapping") + + +class FSMap(MutableMapping): + """Wrap a FileSystem instance as a mutable wrapping. + + The keys of the mapping become files under the given root, and the + values (which must be bytes) the contents of those files. + + Parameters + ---------- + root: string + prefix for all the files + fs: FileSystem instance + check: bool (=True) + performs a touch at the location, to check for write access. + + Examples + -------- + >>> fs = FileSystem(**parameters) # doctest: +SKIP + >>> d = FSMap('my-data/path/', fs) # doctest: +SKIP + or, more likely + >>> d = fs.get_mapper('my-data/path/') + + >>> d['loc1'] = b'Hello World' # doctest: +SKIP + >>> list(d.keys()) # doctest: +SKIP + ['loc1'] + >>> d['loc1'] # doctest: +SKIP + b'Hello World' + """ + + def __init__(self, root, fs, check=False, create=False, missing_exceptions=None): + self.fs = fs + self.root = fs._strip_protocol(root) + self._root_key_to_str = fs._strip_protocol(posixpath.join(root, "x"))[:-1] + if missing_exceptions is None: + missing_exceptions = ( + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + ) + self.missing_exceptions = missing_exceptions + self.check = check + self.create = create + if create: + if not self.fs.exists(root): + self.fs.mkdir(root) + if check: + if not self.fs.exists(root): + raise ValueError( + f"Path {root} does not exist. Create " + f" with the ``create=True`` keyword" + ) + self.fs.touch(root + "/a") + self.fs.rm(root + "/a") + + @cached_property + def dirfs(self): + """dirfs instance that can be used with the same keys as the mapper""" + from .implementations.dirfs import DirFileSystem + + return DirFileSystem(path=self._root_key_to_str, fs=self.fs) + + def clear(self): + """Remove all keys below root - empties out mapping""" + logger.info("Clear mapping at %s", self.root) + try: + self.fs.rm(self.root, True) + self.fs.mkdir(self.root) + except: # noqa: E722 + pass + + def getitems(self, keys, on_error="raise"): + """Fetch multiple items from the store + + If the backend is async-able, this might proceed concurrently + + Parameters + ---------- + keys: list(str) + They keys to be fetched + on_error : "raise", "omit", "return" + If raise, an underlying exception will be raised (converted to KeyError + if the type is in self.missing_exceptions); if omit, keys with exception + will simply not be included in the output; if "return", all keys are + included in the output, but the value will be bytes or an exception + instance. + + Returns + ------- + dict(key, bytes|exception) + """ + keys2 = [self._key_to_str(k) for k in keys] + oe = on_error if on_error == "raise" else "return" + try: + out = self.fs.cat(keys2, on_error=oe) + if isinstance(out, bytes): + out = {keys2[0]: out} + except self.missing_exceptions as e: + raise KeyError from e + out = { + k: (KeyError() if isinstance(v, self.missing_exceptions) else v) + for k, v in out.items() + } + return { + key: out[k2] if on_error == "raise" else out.get(k2, KeyError(k2)) + for key, k2 in zip(keys, keys2) + if on_error == "return" or not isinstance(out[k2], BaseException) + } + + def setitems(self, values_dict): + """Set the values of multiple items in the store + + Parameters + ---------- + values_dict: dict(str, bytes) + """ + values = {self._key_to_str(k): maybe_convert(v) for k, v in values_dict.items()} + self.fs.pipe(values) + + def delitems(self, keys): + """Remove multiple keys from the store""" + self.fs.rm([self._key_to_str(k) for k in keys]) + + def _key_to_str(self, key): + """Generate full path for the key""" + if not isinstance(key, str): + # raise TypeError("key must be of type `str`, got `{type(key).__name__}`" + warnings.warn( + "from fsspec 2023.5 onward FSMap non-str keys will raise TypeError", + DeprecationWarning, + ) + if isinstance(key, list): + key = tuple(key) + key = str(key) + return f"{self._root_key_to_str}{key}".rstrip("/") + + def _str_to_key(self, s): + """Strip path of to leave key name""" + return s[len(self.root) :].lstrip("/") + + def __getitem__(self, key, default=None): + """Retrieve data""" + k = self._key_to_str(key) + try: + result = self.fs.cat(k) + except self.missing_exceptions as exc: + if default is not None: + return default + raise KeyError(key) from exc + return result + + def pop(self, key, default=None): + """Pop data""" + result = self.__getitem__(key, default) + try: + del self[key] + except KeyError: + pass + return result + + def __setitem__(self, key, value): + """Store value in key""" + key = self._key_to_str(key) + self.fs.mkdirs(self.fs._parent(key), exist_ok=True) + self.fs.pipe_file(key, maybe_convert(value)) + + def __iter__(self): + return (self._str_to_key(x) for x in self.fs.find(self.root)) + + def __len__(self): + return len(self.fs.find(self.root)) + + def __delitem__(self, key): + """Remove key""" + try: + self.fs.rm(self._key_to_str(key)) + except Exception as exc: + raise KeyError from exc + + def __contains__(self, key): + """Does key exist in mapping?""" + path = self._key_to_str(key) + return self.fs.isfile(path) + + def __reduce__(self): + return FSMap, (self.root, self.fs, False, False, self.missing_exceptions) + + +def maybe_convert(value): + if isinstance(value, array.array) or hasattr(value, "__array__"): + # bytes-like things + if hasattr(value, "dtype") and value.dtype.kind in "Mm": + # The buffer interface doesn't support datetime64/timdelta64 numpy + # arrays + value = value.view("int64") + value = bytes(memoryview(value)) + return value + + +def get_mapper( + url="", + check=False, + create=False, + missing_exceptions=None, + alternate_root=None, + **kwargs, +): + """Create key-value interface for given URL and options + + The URL will be of the form "protocol://location" and point to the root + of the mapper required. All keys will be file-names below this location, + and their values the contents of each key. + + Also accepts compound URLs like zip::s3://bucket/file.zip , see ``fsspec.open``. + + Parameters + ---------- + url: str + Root URL of mapping + check: bool + Whether to attempt to read from the location before instantiation, to + check that the mapping does exist + create: bool + Whether to make the directory corresponding to the root before + instantiating + missing_exceptions: None or tuple + If given, these exception types will be regarded as missing keys and + return KeyError when trying to read data. By default, you get + (FileNotFoundError, IsADirectoryError, NotADirectoryError) + alternate_root: None or str + In cases of complex URLs, the parser may fail to pick the correct part + for the mapper root, so this arg can override + + Returns + ------- + ``FSMap`` instance, the dict-like key-value store. + """ + # Removing protocol here - could defer to each open() on the backend + fs, urlpath = url_to_fs(url, **kwargs) + root = alternate_root if alternate_root is not None else urlpath + return FSMap(root, fs, check, create, missing_exceptions=missing_exceptions) diff --git a/lib/python3.12/site-packages/fsspec/parquet.py b/lib/python3.12/site-packages/fsspec/parquet.py new file mode 100644 index 0000000000000000000000000000000000000000..faedb7b9e0aa90b6fb7cba33e794c4b4fb35eb77 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/parquet.py @@ -0,0 +1,541 @@ +import io +import json +import warnings + +from .core import url_to_fs +from .utils import merge_offset_ranges + +# Parquet-Specific Utilities for fsspec +# +# Most of the functions defined in this module are NOT +# intended for public consumption. The only exception +# to this is `open_parquet_file`, which should be used +# place of `fs.open()` to open parquet-formatted files +# on remote file systems. + + +def open_parquet_file( + path, + mode="rb", + fs=None, + metadata=None, + columns=None, + row_groups=None, + storage_options=None, + strict=False, + engine="auto", + max_gap=64_000, + max_block=256_000_000, + footer_sample_size=1_000_000, + **kwargs, +): + """ + Return a file-like object for a single Parquet file. + + The specified parquet `engine` will be used to parse the + footer metadata, and determine the required byte ranges + from the file. The target path will then be opened with + the "parts" (`KnownPartsOfAFile`) caching strategy. + + Note that this method is intended for usage with remote + file systems, and is unlikely to improve parquet-read + performance on local file systems. + + Parameters + ---------- + path: str + Target file path. + mode: str, optional + Mode option to be passed through to `fs.open`. Default is "rb". + metadata: Any, optional + Parquet metadata object. Object type must be supported + by the backend parquet engine. For now, only the "fastparquet" + engine supports an explicit `ParquetFile` metadata object. + If a metadata object is supplied, the remote footer metadata + will not need to be transferred into local memory. + fs: AbstractFileSystem, optional + Filesystem object to use for opening the file. If nothing is + specified, an `AbstractFileSystem` object will be inferred. + engine : str, default "auto" + Parquet engine to use for metadata parsing. Allowed options + include "fastparquet", "pyarrow", and "auto". The specified + engine must be installed in the current environment. If + "auto" is specified, and both engines are installed, + "fastparquet" will take precedence over "pyarrow". + columns: list, optional + List of all column names that may be read from the file. + row_groups : list, optional + List of all row-groups that may be read from the file. This + may be a list of row-group indices (integers), or it may be + a list of `RowGroup` metadata objects (if the "fastparquet" + engine is used). + storage_options : dict, optional + Used to generate an `AbstractFileSystem` object if `fs` was + not specified. + strict : bool, optional + Whether the resulting `KnownPartsOfAFile` cache should + fetch reads that go beyond a known byte-range boundary. + If `False` (the default), any read that ends outside a + known part will be zero padded. Note that using + `strict=True` may be useful for debugging. + max_gap : int, optional + Neighboring byte ranges will only be merged when their + inter-range gap is <= `max_gap`. Default is 64KB. + max_block : int, optional + Neighboring byte ranges will only be merged when the size of + the aggregated range is <= `max_block`. Default is 256MB. + footer_sample_size : int, optional + Number of bytes to read from the end of the path to look + for the footer metadata. If the sampled bytes do not contain + the footer, a second read request will be required, and + performance will suffer. Default is 1MB. + **kwargs : + Optional key-word arguments to pass to `fs.open` + """ + + # Make sure we have an `AbstractFileSystem` object + # to work with + if fs is None: + fs = url_to_fs(path, **(storage_options or {}))[0] + + # For now, `columns == []` not supported. Just use + # default `open` command with `path` input + if columns is not None and len(columns) == 0: + return fs.open(path, mode=mode) + + # Set the engine + engine = _set_engine(engine) + + # Fetch the known byte ranges needed to read + # `columns` and/or `row_groups` + data = _get_parquet_byte_ranges( + [path], + fs, + metadata=metadata, + columns=columns, + row_groups=row_groups, + engine=engine, + max_gap=max_gap, + max_block=max_block, + footer_sample_size=footer_sample_size, + ) + + # Extract file name from `data` + fn = next(iter(data)) if data else path + + # Call self.open with "parts" caching + options = kwargs.pop("cache_options", {}).copy() + return fs.open( + fn, + mode=mode, + cache_type="parts", + cache_options={ + **options, + "data": data.get(fn, {}), + "strict": strict, + }, + **kwargs, + ) + + +def _get_parquet_byte_ranges( + paths, + fs, + metadata=None, + columns=None, + row_groups=None, + max_gap=64_000, + max_block=256_000_000, + footer_sample_size=1_000_000, + engine="auto", +): + """Get a dictionary of the known byte ranges needed + to read a specific column/row-group selection from a + Parquet dataset. Each value in the output dictionary + is intended for use as the `data` argument for the + `KnownPartsOfAFile` caching strategy of a single path. + """ + + # Set engine if necessary + if isinstance(engine, str): + engine = _set_engine(engine) + + # Pass to specialized function if metadata is defined + if metadata is not None: + # Use the provided parquet metadata object + # to avoid transferring/parsing footer metadata + return _get_parquet_byte_ranges_from_metadata( + metadata, + fs, + engine, + columns=columns, + row_groups=row_groups, + max_gap=max_gap, + max_block=max_block, + ) + + # Get file sizes asynchronously + file_sizes = fs.sizes(paths) + + # Populate global paths, starts, & ends + result = {} + data_paths = [] + data_starts = [] + data_ends = [] + add_header_magic = True + if columns is None and row_groups is None: + # We are NOT selecting specific columns or row-groups. + # + # We can avoid sampling the footers, and just transfer + # all file data with cat_ranges + for i, path in enumerate(paths): + result[path] = {} + for b in range(0, file_sizes[i], max_block): + data_paths.append(path) + data_starts.append(b) + data_ends.append(min(b + max_block, file_sizes[i])) + add_header_magic = False # "Magic" should already be included + else: + # We ARE selecting specific columns or row-groups. + # + # Gather file footers. + # We just take the last `footer_sample_size` bytes of each + # file (or the entire file if it is smaller than that) + footer_starts = [] + footer_ends = [] + for i, path in enumerate(paths): + footer_ends.append(file_sizes[i]) + sample_size = max(0, file_sizes[i] - footer_sample_size) + footer_starts.append(sample_size) + footer_samples = fs.cat_ranges(paths, footer_starts, footer_ends) + + # Check our footer samples and re-sample if necessary. + missing_footer_starts = footer_starts.copy() + large_footer = 0 + for i, path in enumerate(paths): + footer_size = int.from_bytes(footer_samples[i][-8:-4], "little") + real_footer_start = file_sizes[i] - (footer_size + 8) + if real_footer_start < footer_starts[i]: + missing_footer_starts[i] = real_footer_start + large_footer = max(large_footer, (footer_size + 8)) + if large_footer: + warnings.warn( + f"Not enough data was used to sample the parquet footer. " + f"Try setting footer_sample_size >= {large_footer}." + ) + for i, block in enumerate( + fs.cat_ranges( + paths, + missing_footer_starts, + footer_starts, + ) + ): + footer_samples[i] = block + footer_samples[i] + footer_starts[i] = missing_footer_starts[i] + + # Calculate required byte ranges for each path + for i, path in enumerate(paths): + # Deal with small-file case. + # Just include all remaining bytes of the file + # in a single range. + if file_sizes[i] < max_block: + if footer_starts[i] > 0: + # Only need to transfer the data if the + # footer sample isn't already the whole file + data_paths.append(path) + data_starts.append(0) + data_ends.append(footer_starts[i]) + continue + + # Use "engine" to collect data byte ranges + path_data_starts, path_data_ends = engine._parquet_byte_ranges( + columns, + row_groups=row_groups, + footer=footer_samples[i], + footer_start=footer_starts[i], + ) + + data_paths += [path] * len(path_data_starts) + data_starts += path_data_starts + data_ends += path_data_ends + + # Merge adjacent offset ranges + data_paths, data_starts, data_ends = merge_offset_ranges( + data_paths, + data_starts, + data_ends, + max_gap=max_gap, + max_block=max_block, + sort=False, # Should already be sorted + ) + + # Start by populating `result` with footer samples + for i, path in enumerate(paths): + result[path] = {(footer_starts[i], footer_ends[i]): footer_samples[i]} + + # Transfer the data byte-ranges into local memory + _transfer_ranges(fs, result, data_paths, data_starts, data_ends) + + # Add b"PAR1" to header if necessary + if add_header_magic: + _add_header_magic(result) + + return result + + +def _get_parquet_byte_ranges_from_metadata( + metadata, + fs, + engine, + columns=None, + row_groups=None, + max_gap=64_000, + max_block=256_000_000, +): + """Simplified version of `_get_parquet_byte_ranges` for + the case that an engine-specific `metadata` object is + provided, and the remote footer metadata does not need to + be transferred before calculating the required byte ranges. + """ + + # Use "engine" to collect data byte ranges + data_paths, data_starts, data_ends = engine._parquet_byte_ranges( + columns, + row_groups=row_groups, + metadata=metadata, + ) + + # Merge adjacent offset ranges + data_paths, data_starts, data_ends = merge_offset_ranges( + data_paths, + data_starts, + data_ends, + max_gap=max_gap, + max_block=max_block, + sort=False, # Should be sorted + ) + + # Transfer the data byte-ranges into local memory + result = {fn: {} for fn in list(set(data_paths))} + _transfer_ranges(fs, result, data_paths, data_starts, data_ends) + + # Add b"PAR1" to header + _add_header_magic(result) + + return result + + +def _transfer_ranges(fs, blocks, paths, starts, ends): + # Use cat_ranges to gather the data byte_ranges + ranges = (paths, starts, ends) + for path, start, stop, data in zip(*ranges, fs.cat_ranges(*ranges)): + blocks[path][(start, stop)] = data + + +def _add_header_magic(data): + # Add b"PAR1" to file headers + for path in list(data.keys()): + add_magic = True + for k in data[path]: + if k[0] == 0 and k[1] >= 4: + add_magic = False + break + if add_magic: + data[path][(0, 4)] = b"PAR1" + + +def _set_engine(engine_str): + # Define a list of parquet engines to try + if engine_str == "auto": + try_engines = ("fastparquet", "pyarrow") + elif not isinstance(engine_str, str): + raise ValueError( + "Failed to set parquet engine! " + "Please pass 'fastparquet', 'pyarrow', or 'auto'" + ) + elif engine_str not in ("fastparquet", "pyarrow"): + raise ValueError(f"{engine_str} engine not supported by `fsspec.parquet`") + else: + try_engines = [engine_str] + + # Try importing the engines in `try_engines`, + # and choose the first one that succeeds + for engine in try_engines: + try: + if engine == "fastparquet": + return FastparquetEngine() + elif engine == "pyarrow": + return PyarrowEngine() + except ImportError: + pass + + # Raise an error if a supported parquet engine + # was not found + raise ImportError( + f"The following parquet engines are not installed " + f"in your python environment: {try_engines}." + f"Please install 'fastparquert' or 'pyarrow' to " + f"utilize the `fsspec.parquet` module." + ) + + +class FastparquetEngine: + # The purpose of the FastparquetEngine class is + # to check if fastparquet can be imported (on initialization) + # and to define a `_parquet_byte_ranges` method. In the + # future, this class may also be used to define other + # methods/logic that are specific to fastparquet. + + def __init__(self): + import fastparquet as fp + + self.fp = fp + + def _row_group_filename(self, row_group, pf): + return pf.row_group_filename(row_group) + + def _parquet_byte_ranges( + self, + columns, + row_groups=None, + metadata=None, + footer=None, + footer_start=None, + ): + # Initialize offset ranges and define ParqetFile metadata + pf = metadata + data_paths, data_starts, data_ends = [], [], [] + if pf is None: + pf = self.fp.ParquetFile(io.BytesIO(footer)) + + # Convert columns to a set and add any index columns + # specified in the pandas metadata (just in case) + column_set = None if columns is None else set(columns) + if column_set is not None and hasattr(pf, "pandas_metadata"): + md_index = [ + ind + for ind in pf.pandas_metadata.get("index_columns", []) + # Ignore RangeIndex information + if not isinstance(ind, dict) + ] + column_set |= set(md_index) + + # Check if row_groups is a list of integers + # or a list of row-group metadata + if row_groups and not isinstance(row_groups[0], int): + # Input row_groups contains row-group metadata + row_group_indices = None + else: + # Input row_groups contains row-group indices + row_group_indices = row_groups + row_groups = pf.row_groups + + # Loop through column chunks to add required byte ranges + for r, row_group in enumerate(row_groups): + # Skip this row-group if we are targeting + # specific row-groups + if row_group_indices is None or r in row_group_indices: + # Find the target parquet-file path for `row_group` + fn = self._row_group_filename(row_group, pf) + + for column in row_group.columns: + name = column.meta_data.path_in_schema[0] + # Skip this column if we are targeting a + # specific columns + if column_set is None or name in column_set: + file_offset0 = column.meta_data.dictionary_page_offset + if file_offset0 is None: + file_offset0 = column.meta_data.data_page_offset + num_bytes = column.meta_data.total_compressed_size + if footer_start is None or file_offset0 < footer_start: + data_paths.append(fn) + data_starts.append(file_offset0) + data_ends.append( + min( + file_offset0 + num_bytes, + footer_start or (file_offset0 + num_bytes), + ) + ) + + if metadata: + # The metadata in this call may map to multiple + # file paths. Need to include `data_paths` + return data_paths, data_starts, data_ends + return data_starts, data_ends + + +class PyarrowEngine: + # The purpose of the PyarrowEngine class is + # to check if pyarrow can be imported (on initialization) + # and to define a `_parquet_byte_ranges` method. In the + # future, this class may also be used to define other + # methods/logic that are specific to pyarrow. + + def __init__(self): + import pyarrow.parquet as pq + + self.pq = pq + + def _row_group_filename(self, row_group, metadata): + raise NotImplementedError + + def _parquet_byte_ranges( + self, + columns, + row_groups=None, + metadata=None, + footer=None, + footer_start=None, + ): + if metadata is not None: + raise ValueError("metadata input not supported for PyarrowEngine") + + data_starts, data_ends = [], [] + md = self.pq.ParquetFile(io.BytesIO(footer)).metadata + + # Convert columns to a set and add any index columns + # specified in the pandas metadata (just in case) + column_set = None if columns is None else set(columns) + if column_set is not None: + schema = md.schema.to_arrow_schema() + has_pandas_metadata = ( + schema.metadata is not None and b"pandas" in schema.metadata + ) + if has_pandas_metadata: + md_index = [ + ind + for ind in json.loads( + schema.metadata[b"pandas"].decode("utf8") + ).get("index_columns", []) + # Ignore RangeIndex information + if not isinstance(ind, dict) + ] + column_set |= set(md_index) + + # Loop through column chunks to add required byte ranges + for r in range(md.num_row_groups): + # Skip this row-group if we are targeting + # specific row-groups + if row_groups is None or r in row_groups: + row_group = md.row_group(r) + for c in range(row_group.num_columns): + column = row_group.column(c) + name = column.path_in_schema + # Skip this column if we are targeting a + # specific columns + split_name = name.split(".")[0] + if ( + column_set is None + or name in column_set + or split_name in column_set + ): + file_offset0 = column.dictionary_page_offset + if file_offset0 is None: + file_offset0 = column.data_page_offset + num_bytes = column.total_compressed_size + if file_offset0 < footer_start: + data_starts.append(file_offset0) + data_ends.append( + min(file_offset0 + num_bytes, footer_start) + ) + return data_starts, data_ends diff --git a/lib/python3.12/site-packages/fsspec/registry.py b/lib/python3.12/site-packages/fsspec/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..59be4db7b6773fdeffba8e3c517985557bb802c6 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/registry.py @@ -0,0 +1,318 @@ +from __future__ import annotations + +import importlib +import types +import warnings + +__all__ = ["registry", "get_filesystem_class", "default"] + +# internal, mutable +_registry: dict[str, type] = {} + +# external, immutable +registry = types.MappingProxyType(_registry) +default = "file" + + +def register_implementation(name, cls, clobber=False, errtxt=None): + """Add implementation class to the registry + + Parameters + ---------- + name: str + Protocol name to associate with the class + cls: class or str + if a class: fsspec-compliant implementation class (normally inherits from + ``fsspec.AbstractFileSystem``, gets added straight to the registry. If a + str, the full path to an implementation class like package.module.class, + which gets added to known_implementations, + so the import is deferred until the filesystem is actually used. + clobber: bool (optional) + Whether to overwrite a protocol with the same name; if False, will raise + instead. + errtxt: str (optional) + If given, then a failure to import the given class will result in this + text being given. + """ + if isinstance(cls, str): + if name in known_implementations and clobber is False: + if cls != known_implementations[name]["class"]: + raise ValueError( + f"Name ({name}) already in the known_implementations and clobber " + f"is False" + ) + else: + known_implementations[name] = { + "class": cls, + "err": errtxt or f"{cls} import failed for protocol {name}", + } + + else: + if name in registry and clobber is False: + if _registry[name] is not cls: + raise ValueError( + f"Name ({name}) already in the registry and clobber is False" + ) + else: + _registry[name] = cls + + +# protocols mapped to the class which implements them. This dict can be +# updated with register_implementation +known_implementations = { + "abfs": { + "class": "adlfs.AzureBlobFileSystem", + "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", + }, + "adl": { + "class": "adlfs.AzureDatalakeFileSystem", + "err": "Install adlfs to access Azure Datalake Gen1", + }, + "arrow_hdfs": { + "class": "fsspec.implementations.arrow.HadoopFileSystem", + "err": "pyarrow and local java libraries required for HDFS", + }, + "async_wrapper": { + "class": "morefs.asyn_wrapper.AsyncWrapperFileSystem", + }, + "asynclocal": { + "class": "morefs.asyn_local.AsyncLocalFileSystem", + "err": "Install 'morefs[asynclocalfs]' to use AsyncLocalFileSystem", + }, + "az": { + "class": "adlfs.AzureBlobFileSystem", + "err": "Install adlfs to access Azure Datalake Gen2 and Azure Blob Storage", + }, + "blockcache": {"class": "fsspec.implementations.cached.CachingFileSystem"}, + "box": { + "class": "boxfs.BoxFileSystem", + "err": "Please install boxfs to access BoxFileSystem", + }, + "cached": {"class": "fsspec.implementations.cached.CachingFileSystem"}, + "dask": { + "class": "fsspec.implementations.dask.DaskWorkerFileSystem", + "err": "Install dask distributed to access worker file system", + }, + "data": {"class": "fsspec.implementations.data.DataFileSystem"}, + "dbfs": { + "class": "fsspec.implementations.dbfs.DatabricksFileSystem", + "err": "Install the requests package to use the DatabricksFileSystem", + }, + "dir": {"class": "fsspec.implementations.dirfs.DirFileSystem"}, + "dropbox": { + "class": "dropboxdrivefs.DropboxDriveFileSystem", + "err": ( + 'DropboxFileSystem requires "dropboxdrivefs","requests" and "' + '"dropbox" to be installed' + ), + }, + "dvc": { + "class": "dvc.api.DVCFileSystem", + "err": "Install dvc to access DVCFileSystem", + }, + "file": {"class": "fsspec.implementations.local.LocalFileSystem"}, + "filecache": {"class": "fsspec.implementations.cached.WholeFileCacheFileSystem"}, + "ftp": {"class": "fsspec.implementations.ftp.FTPFileSystem"}, + "gcs": { + "class": "gcsfs.GCSFileSystem", + "err": "Please install gcsfs to access Google Storage", + }, + "gdrive": { + "class": "gdrivefs.GoogleDriveFileSystem", + "err": "Please install gdrivefs for access to Google Drive", + }, + "generic": {"class": "fsspec.generic.GenericFileSystem"}, + "git": { + "class": "fsspec.implementations.git.GitFileSystem", + "err": "Install pygit2 to browse local git repos", + }, + "github": { + "class": "fsspec.implementations.github.GithubFileSystem", + "err": "Install the requests package to use the github FS", + }, + "gs": { + "class": "gcsfs.GCSFileSystem", + "err": "Please install gcsfs to access Google Storage", + }, + "hdfs": { + "class": "fsspec.implementations.arrow.HadoopFileSystem", + "err": "pyarrow and local java libraries required for HDFS", + }, + "hf": { + "class": "huggingface_hub.HfFileSystem", + "err": "Install huggingface_hub to access HfFileSystem", + }, + "http": { + "class": "fsspec.implementations.http.HTTPFileSystem", + "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', + }, + "https": { + "class": "fsspec.implementations.http.HTTPFileSystem", + "err": 'HTTPFileSystem requires "requests" and "aiohttp" to be installed', + }, + "jlab": { + "class": "fsspec.implementations.jupyter.JupyterFileSystem", + "err": "Jupyter FS requires requests to be installed", + }, + "jupyter": { + "class": "fsspec.implementations.jupyter.JupyterFileSystem", + "err": "Jupyter FS requires requests to be installed", + }, + "lakefs": { + "class": "lakefs_spec.LakeFSFileSystem", + "err": "Please install lakefs-spec to access LakeFSFileSystem", + }, + "libarchive": { + "class": "fsspec.implementations.libarchive.LibArchiveFileSystem", + "err": "LibArchive requires to be installed", + }, + "local": {"class": "fsspec.implementations.local.LocalFileSystem"}, + "memory": {"class": "fsspec.implementations.memory.MemoryFileSystem"}, + "oci": { + "class": "ocifs.OCIFileSystem", + "err": "Install ocifs to access OCI Object Storage", + }, + "ocilake": { + "class": "ocifs.OCIFileSystem", + "err": "Install ocifs to access OCI Data Lake", + }, + "oss": { + "class": "ossfs.OSSFileSystem", + "err": "Install ossfs to access Alibaba Object Storage System", + }, + "reference": {"class": "fsspec.implementations.reference.ReferenceFileSystem"}, + "root": { + "class": "fsspec_xrootd.XRootDFileSystem", + "err": ( + "Install fsspec-xrootd to access xrootd storage system. " + "Note: 'root' is the protocol name for xrootd storage systems, " + "not referring to root directories" + ), + }, + "s3": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, + "s3a": {"class": "s3fs.S3FileSystem", "err": "Install s3fs to access S3"}, + "sftp": { + "class": "fsspec.implementations.sftp.SFTPFileSystem", + "err": 'SFTPFileSystem requires "paramiko" to be installed', + }, + "simplecache": {"class": "fsspec.implementations.cached.SimpleCacheFileSystem"}, + "smb": { + "class": "fsspec.implementations.smb.SMBFileSystem", + "err": 'SMB requires "smbprotocol" or "smbprotocol[kerberos]" installed', + }, + "ssh": { + "class": "fsspec.implementations.sftp.SFTPFileSystem", + "err": 'SFTPFileSystem requires "paramiko" to be installed', + }, + "tar": {"class": "fsspec.implementations.tar.TarFileSystem"}, + "tosfs": { + "class": "tosfs.TosFileSystem", + "err": "Install tosfs to access ByteDance volcano engine Tinder Object Storage", + }, + "wandb": {"class": "wandbfs.WandbFS", "err": "Install wandbfs to access wandb"}, + "webdav": { + "class": "webdav4.fsspec.WebdavFileSystem", + "err": "Install webdav4 to access WebDAV", + }, + "webhdfs": { + "class": "fsspec.implementations.webhdfs.WebHDFS", + "err": 'webHDFS access requires "requests" to be installed', + }, + "zip": {"class": "fsspec.implementations.zip.ZipFileSystem"}, +} + +assert list(known_implementations) == sorted(known_implementations), ( + "Not in alphabetical order" +) + + +def get_filesystem_class(protocol): + """Fetch named protocol implementation from the registry + + The dict ``known_implementations`` maps protocol names to the locations + of classes implementing the corresponding file-system. When used for the + first time, appropriate imports will happen and the class will be placed in + the registry. All subsequent calls will fetch directly from the registry. + + Some protocol implementations require additional dependencies, and so the + import may fail. In this case, the string in the "err" field of the + ``known_implementations`` will be given as the error message. + """ + if not protocol: + protocol = default + + if protocol not in registry: + if protocol not in known_implementations: + raise ValueError(f"Protocol not known: {protocol}") + bit = known_implementations[protocol] + try: + register_implementation(protocol, _import_class(bit["class"])) + except ImportError as e: + raise ImportError(bit["err"]) from e + cls = registry[protocol] + if getattr(cls, "protocol", None) in ("abstract", None): + cls.protocol = protocol + + return cls + + +s3_msg = """Your installed version of s3fs is very old and known to cause +severe performance issues, see also https://github.com/dask/dask/issues/10276 + +To fix, you should specify a lower version bound on s3fs, or +update the current installation. +""" + + +def _import_class(fqp: str): + """Take a fully-qualified path and return the imported class or identifier. + + ``fqp`` is of the form "package.module.klass" or + "package.module:subobject.klass". + + Warnings + -------- + This can import arbitrary modules. Make sure you haven't installed any modules + that may execute malicious code at import time. + """ + if ":" in fqp: + mod, name = fqp.rsplit(":", 1) + else: + mod, name = fqp.rsplit(".", 1) + + is_s3 = mod == "s3fs" + mod = importlib.import_module(mod) + if is_s3 and mod.__version__.split(".") < ["0", "5"]: + warnings.warn(s3_msg) + for part in name.split("."): + mod = getattr(mod, part) + + if not isinstance(mod, type): + raise TypeError(f"{fqp} is not a class") + + return mod + + +def filesystem(protocol, **storage_options): + """Instantiate filesystems for given protocol and arguments + + ``storage_options`` are specific to the protocol being chosen, and are + passed directly to the class. + """ + if protocol == "arrow_hdfs": + warnings.warn( + "The 'arrow_hdfs' protocol has been deprecated and will be " + "removed in the future. Specify it as 'hdfs'.", + DeprecationWarning, + ) + + cls = get_filesystem_class(protocol) + return cls(**storage_options) + + +def available_protocols(): + """Return a list of the implemented protocols. + + Note that any given protocol may require extra packages to be importable. + """ + return list(known_implementations) diff --git a/lib/python3.12/site-packages/fsspec/spec.py b/lib/python3.12/site-packages/fsspec/spec.py new file mode 100644 index 0000000000000000000000000000000000000000..d83cb099d02baca9d3fb665cbe992f22f3e0c1d0 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/spec.py @@ -0,0 +1,2242 @@ +from __future__ import annotations + +import io +import json +import logging +import os +import threading +import warnings +import weakref +from errno import ESPIPE +from glob import has_magic +from hashlib import sha256 +from typing import Any, ClassVar + +from .callbacks import DEFAULT_CALLBACK +from .config import apply_config, conf +from .dircache import DirCache +from .transaction import Transaction +from .utils import ( + _unstrip_protocol, + glob_translate, + isfilelike, + other_paths, + read_block, + stringify_path, + tokenize, +) + +logger = logging.getLogger("fsspec") + + +def make_instance(cls, args, kwargs): + return cls(*args, **kwargs) + + +class _Cached(type): + """ + Metaclass for caching file system instances. + + Notes + ----- + Instances are cached according to + + * The values of the class attributes listed in `_extra_tokenize_attributes` + * The arguments passed to ``__init__``. + + This creates an additional reference to the filesystem, which prevents the + filesystem from being garbage collected when all *user* references go away. + A call to the :meth:`AbstractFileSystem.clear_instance_cache` must *also* + be made for a filesystem instance to be garbage collected. + """ + + def __init__(cls, *args, **kwargs): + super().__init__(*args, **kwargs) + # Note: we intentionally create a reference here, to avoid garbage + # collecting instances when all other references are gone. To really + # delete a FileSystem, the cache must be cleared. + if conf.get("weakref_instance_cache"): # pragma: no cover + # debug option for analysing fork/spawn conditions + cls._cache = weakref.WeakValueDictionary() + else: + cls._cache = {} + cls._pid = os.getpid() + + def __call__(cls, *args, **kwargs): + kwargs = apply_config(cls, kwargs) + extra_tokens = tuple( + getattr(cls, attr, None) for attr in cls._extra_tokenize_attributes + ) + token = tokenize( + cls, cls._pid, threading.get_ident(), *args, *extra_tokens, **kwargs + ) + skip = kwargs.pop("skip_instance_cache", False) + if os.getpid() != cls._pid: + cls._cache.clear() + cls._pid = os.getpid() + if not skip and cls.cachable and token in cls._cache: + cls._latest = token + return cls._cache[token] + else: + obj = super().__call__(*args, **kwargs) + # Setting _fs_token here causes some static linters to complain. + obj._fs_token_ = token + obj.storage_args = args + obj.storage_options = kwargs + if obj.async_impl and obj.mirror_sync_methods: + from .asyn import mirror_sync_methods + + mirror_sync_methods(obj) + + if cls.cachable and not skip: + cls._latest = token + cls._cache[token] = obj + return obj + + +class AbstractFileSystem(metaclass=_Cached): + """ + An abstract super-class for pythonic file-systems + + Implementations are expected to be compatible with or, better, subclass + from here. + """ + + cachable = True # this class can be cached, instances reused + _cached = False + blocksize = 2**22 + sep = "/" + protocol: ClassVar[str | tuple[str, ...]] = "abstract" + _latest = None + async_impl = False + mirror_sync_methods = False + root_marker = "" # For some FSs, may require leading '/' or other character + transaction_type = Transaction + + #: Extra *class attributes* that should be considered when hashing. + _extra_tokenize_attributes = () + + # Set by _Cached metaclass + storage_args: tuple[Any, ...] + storage_options: dict[str, Any] + + def __init__(self, *args, **storage_options): + """Create and configure file-system instance + + Instances may be cachable, so if similar enough arguments are seen + a new instance is not required. The token attribute exists to allow + implementations to cache instances if they wish. + + A reasonable default should be provided if there are no arguments. + + Subclasses should call this method. + + Parameters + ---------- + use_listings_cache, listings_expiry_time, max_paths: + passed to ``DirCache``, if the implementation supports + directory listing caching. Pass use_listings_cache=False + to disable such caching. + skip_instance_cache: bool + If this is a cachable implementation, pass True here to force + creating a new instance even if a matching instance exists, and prevent + storing this instance. + asynchronous: bool + loop: asyncio-compatible IOLoop or None + """ + if self._cached: + # reusing instance, don't change + return + self._cached = True + self._intrans = False + self._transaction = None + self._invalidated_caches_in_transaction = [] + self.dircache = DirCache(**storage_options) + + if storage_options.pop("add_docs", None): + warnings.warn("add_docs is no longer supported.", FutureWarning) + + if storage_options.pop("add_aliases", None): + warnings.warn("add_aliases has been removed.", FutureWarning) + # This is set in _Cached + self._fs_token_ = None + + @property + def fsid(self): + """Persistent filesystem id that can be used to compare filesystems + across sessions. + """ + raise NotImplementedError + + @property + def _fs_token(self): + return self._fs_token_ + + def __dask_tokenize__(self): + return self._fs_token + + def __hash__(self): + return int(self._fs_token, 16) + + def __eq__(self, other): + return isinstance(other, type(self)) and self._fs_token == other._fs_token + + def __reduce__(self): + return make_instance, (type(self), self.storage_args, self.storage_options) + + @classmethod + def _strip_protocol(cls, path): + """Turn path from fully-qualified to file-system-specific + + May require FS-specific handling, e.g., for relative paths or links. + """ + if isinstance(path, list): + return [cls._strip_protocol(p) for p in path] + path = stringify_path(path) + protos = (cls.protocol,) if isinstance(cls.protocol, str) else cls.protocol + for protocol in protos: + if path.startswith(protocol + "://"): + path = path[len(protocol) + 3 :] + elif path.startswith(protocol + "::"): + path = path[len(protocol) + 2 :] + path = path.rstrip("/") + # use of root_marker to make minimum required path, e.g., "/" + return path or cls.root_marker + + def unstrip_protocol(self, name: str) -> str: + """Format FS-specific path to generic, including protocol""" + protos = (self.protocol,) if isinstance(self.protocol, str) else self.protocol + for protocol in protos: + if name.startswith(f"{protocol}://"): + return name + return f"{protos[0]}://{name}" + + @staticmethod + def _get_kwargs_from_urls(path): + """If kwargs can be encoded in the paths, extract them here + + This should happen before instantiation of the class; incoming paths + then should be amended to strip the options in methods. + + Examples may look like an sftp path "sftp://user@host:/my/path", where + the user and host should become kwargs and later get stripped. + """ + # by default, nothing happens + return {} + + @classmethod + def current(cls): + """Return the most recently instantiated FileSystem + + If no instance has been created, then create one with defaults + """ + if cls._latest in cls._cache: + return cls._cache[cls._latest] + return cls() + + @property + def transaction(self): + """A context within which files are committed together upon exit + + Requires the file class to implement `.commit()` and `.discard()` + for the normal and exception cases. + """ + if self._transaction is None: + self._transaction = self.transaction_type(self) + return self._transaction + + def start_transaction(self): + """Begin write transaction for deferring files, non-context version""" + self._intrans = True + self._transaction = self.transaction_type(self) + return self.transaction + + def end_transaction(self): + """Finish write transaction, non-context version""" + self.transaction.complete() + self._transaction = None + # The invalid cache must be cleared after the transaction is completed. + for path in self._invalidated_caches_in_transaction: + self.invalidate_cache(path) + self._invalidated_caches_in_transaction.clear() + + def invalidate_cache(self, path=None): + """ + Discard any cached directory information + + Parameters + ---------- + path: string or None + If None, clear all listings cached else listings at or under given + path. + """ + # Not necessary to implement invalidation mechanism, may have no cache. + # But if have, you should call this method of parent class from your + # subclass to ensure expiring caches after transacations correctly. + # See the implementation of FTPFileSystem in ftp.py + if self._intrans: + self._invalidated_caches_in_transaction.append(path) + + def mkdir(self, path, create_parents=True, **kwargs): + """ + Create directory entry at path + + For systems that don't have true directories, may create an for + this instance only and not touch the real filesystem + + Parameters + ---------- + path: str + location + create_parents: bool + if True, this is equivalent to ``makedirs`` + kwargs: + may be permissions, etc. + """ + pass # not necessary to implement, may not have directories + + def makedirs(self, path, exist_ok=False): + """Recursively make directories + + Creates directory at path and any intervening required directories. + Raises exception if, for instance, the path already exists but is a + file. + + Parameters + ---------- + path: str + leaf directory name + exist_ok: bool (False) + If False, will error if the target already exists + """ + pass # not necessary to implement, may not have directories + + def rmdir(self, path): + """Remove a directory, if empty""" + pass # not necessary to implement, may not have directories + + def ls(self, path, detail=True, **kwargs): + """List objects at path. + + This should include subdirectories and files at that location. The + difference between a file and a directory must be clear when details + are requested. + + The specific keys, or perhaps a FileInfo class, or similar, is TBD, + but must be consistent across implementations. + Must include: + + - full path to the entry (without protocol) + - size of the entry, in bytes. If the value cannot be determined, will + be ``None``. + - type of entry, "file", "directory" or other + + Additional information + may be present, appropriate to the file-system, e.g., generation, + checksum, etc. + + May use refresh=True|False to allow use of self._ls_from_cache to + check for a saved listing and avoid calling the backend. This would be + common where listing may be expensive. + + Parameters + ---------- + path: str + detail: bool + if True, gives a list of dictionaries, where each is the same as + the result of ``info(path)``. If False, gives a list of paths + (str). + kwargs: may have additional backend-specific options, such as version + information + + Returns + ------- + List of strings if detail is False, or list of directory information + dicts if detail is True. + """ + raise NotImplementedError + + def _ls_from_cache(self, path): + """Check cache for listing + + Returns listing, if found (may be empty list for a directly that exists + but contains nothing), None if not in cache. + """ + parent = self._parent(path) + try: + return self.dircache[path.rstrip("/")] + except KeyError: + pass + try: + files = [ + f + for f in self.dircache[parent] + if f["name"] == path + or (f["name"] == path.rstrip("/") and f["type"] == "directory") + ] + if len(files) == 0: + # parent dir was listed but did not contain this file + raise FileNotFoundError(path) + return files + except KeyError: + pass + + def walk(self, path, maxdepth=None, topdown=True, on_error="omit", **kwargs): + """Return all files under the given path. + + List all files, recursing into subdirectories; output is iterator-style, + like ``os.walk()``. For a simple list of files, ``find()`` is available. + + When topdown is True, the caller can modify the dirnames list in-place (perhaps + using del or slice assignment), and walk() will + only recurse into the subdirectories whose names remain in dirnames; + this can be used to prune the search, impose a specific order of visiting, + or even to inform walk() about directories the caller creates or renames before + it resumes walk() again. + Modifying dirnames when topdown is False has no effect. (see os.walk) + + Note that the "files" outputted will include anything that is not + a directory, such as links. + + Parameters + ---------- + path: str + Root to recurse into + maxdepth: int + Maximum recursion depth. None means limitless, but not recommended + on link-based file-systems. + topdown: bool (True) + Whether to walk the directory tree from the top downwards or from + the bottom upwards. + on_error: "omit", "raise", a callable + if omit (default), path with exception will simply be empty; + If raise, an underlying exception will be raised; + if callable, it will be called with a single OSError instance as argument + kwargs: passed to ``ls`` + """ + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + path = self._strip_protocol(path) + full_dirs = {} + dirs = {} + files = {} + + detail = kwargs.pop("detail", False) + try: + listing = self.ls(path, detail=True, **kwargs) + except (FileNotFoundError, OSError) as e: + if on_error == "raise": + raise + if callable(on_error): + on_error(e) + return + + for info in listing: + # each info name must be at least [path]/part , but here + # we check also for names like [path]/part/ + pathname = info["name"].rstrip("/") + name = pathname.rsplit("/", 1)[-1] + if info["type"] == "directory" and pathname != path: + # do not include "self" path + full_dirs[name] = pathname + dirs[name] = info + elif pathname == path: + # file-like with same name as give path + files[""] = info + else: + files[name] = info + + if not detail: + dirs = list(dirs) + files = list(files) + + if topdown: + # Yield before recursion if walking top down + yield path, dirs, files + + if maxdepth is not None: + maxdepth -= 1 + if maxdepth < 1: + if not topdown: + yield path, dirs, files + return + + for d in dirs: + yield from self.walk( + full_dirs[d], + maxdepth=maxdepth, + detail=detail, + topdown=topdown, + **kwargs, + ) + + if not topdown: + # Yield after recursion if walking bottom up + yield path, dirs, files + + def find(self, path, maxdepth=None, withdirs=False, detail=False, **kwargs): + """List all files below path. + + Like posix ``find`` command without conditions + + Parameters + ---------- + path : str + maxdepth: int or None + If not None, the maximum number of levels to descend + withdirs: bool + Whether to include directory paths in the output. This is True + when used by glob, but users usually only want files. + kwargs are passed to ``ls``. + """ + # TODO: allow equivalent of -name parameter + path = self._strip_protocol(path) + out = {} + + # Add the root directory if withdirs is requested + # This is needed for posix glob compliance + if withdirs and path != "" and self.isdir(path): + out[path] = self.info(path) + + for _, dirs, files in self.walk(path, maxdepth, detail=True, **kwargs): + if withdirs: + files.update(dirs) + out.update({info["name"]: info for name, info in files.items()}) + if not out and self.isfile(path): + # walk works on directories, but find should also return [path] + # when path happens to be a file + out[path] = {} + names = sorted(out) + if not detail: + return names + else: + return {name: out[name] for name in names} + + def du(self, path, total=True, maxdepth=None, withdirs=False, **kwargs): + """Space used by files and optionally directories within a path + + Directory size does not include the size of its contents. + + Parameters + ---------- + path: str + total: bool + Whether to sum all the file sizes + maxdepth: int or None + Maximum number of directory levels to descend, None for unlimited. + withdirs: bool + Whether to include directory paths in the output. + kwargs: passed to ``find`` + + Returns + ------- + Dict of {path: size} if total=False, or int otherwise, where numbers + refer to bytes used. + """ + sizes = {} + if withdirs and self.isdir(path): + # Include top-level directory in output + info = self.info(path) + sizes[info["name"]] = info["size"] + for f in self.find(path, maxdepth=maxdepth, withdirs=withdirs, **kwargs): + info = self.info(f) + sizes[info["name"]] = info["size"] + if total: + return sum(sizes.values()) + else: + return sizes + + def glob(self, path, maxdepth=None, **kwargs): + """ + Find files by glob-matching. + + If the path ends with '/', only folders are returned. + + We support ``"**"``, + ``"?"`` and ``"[..]"``. We do not support ^ for pattern negation. + + The `maxdepth` option is applied on the first `**` found in the path. + + kwargs are passed to ``ls``. + """ + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + import re + + seps = (os.path.sep, os.path.altsep) if os.path.altsep else (os.path.sep,) + ends_with_sep = path.endswith(seps) # _strip_protocol strips trailing slash + path = self._strip_protocol(path) + append_slash_to_dirname = ends_with_sep or path.endswith( + tuple(sep + "**" for sep in seps) + ) + idx_star = path.find("*") if path.find("*") >= 0 else len(path) + idx_qmark = path.find("?") if path.find("?") >= 0 else len(path) + idx_brace = path.find("[") if path.find("[") >= 0 else len(path) + + min_idx = min(idx_star, idx_qmark, idx_brace) + + detail = kwargs.pop("detail", False) + + if not has_magic(path): + if self.exists(path, **kwargs): + if not detail: + return [path] + else: + return {path: self.info(path, **kwargs)} + else: + if not detail: + return [] # glob of non-existent returns empty + else: + return {} + elif "/" in path[:min_idx]: + min_idx = path[:min_idx].rindex("/") + root = path[: min_idx + 1] + depth = path[min_idx + 1 :].count("/") + 1 + else: + root = "" + depth = path[min_idx + 1 :].count("/") + 1 + + if "**" in path: + if maxdepth is not None: + idx_double_stars = path.find("**") + depth_double_stars = path[idx_double_stars:].count("/") + 1 + depth = depth - depth_double_stars + maxdepth + else: + depth = None + + allpaths = self.find(root, maxdepth=depth, withdirs=True, detail=True, **kwargs) + + pattern = glob_translate(path + ("/" if ends_with_sep else "")) + pattern = re.compile(pattern) + + out = { + p: info + for p, info in sorted(allpaths.items()) + if pattern.match( + p + "/" + if append_slash_to_dirname and info["type"] == "directory" + else p + ) + } + + if detail: + return out + else: + return list(out) + + def exists(self, path, **kwargs): + """Is there a file at the given path""" + try: + self.info(path, **kwargs) + return True + except: # noqa: E722 + # any exception allowed bar FileNotFoundError? + return False + + def lexists(self, path, **kwargs): + """If there is a file at the given path (including + broken links)""" + return self.exists(path) + + def info(self, path, **kwargs): + """Give details of entry at path + + Returns a single dictionary, with exactly the same information as ``ls`` + would with ``detail=True``. + + The default implementation calls ls and could be overridden by a + shortcut. kwargs are passed on to ```ls()``. + + Some file systems might not be able to measure the file's size, in + which case, the returned dict will include ``'size': None``. + + Returns + ------- + dict with keys: name (full path in the FS), size (in bytes), type (file, + directory, or something else) and other FS-specific keys. + """ + path = self._strip_protocol(path) + out = self.ls(self._parent(path), detail=True, **kwargs) + out = [o for o in out if o["name"].rstrip("/") == path] + if out: + return out[0] + out = self.ls(path, detail=True, **kwargs) + path = path.rstrip("/") + out1 = [o for o in out if o["name"].rstrip("/") == path] + if len(out1) == 1: + if "size" not in out1[0]: + out1[0]["size"] = None + return out1[0] + elif len(out1) > 1 or out: + return {"name": path, "size": 0, "type": "directory"} + else: + raise FileNotFoundError(path) + + def checksum(self, path): + """Unique value for current version of file + + If the checksum is the same from one moment to another, the contents + are guaranteed to be the same. If the checksum changes, the contents + *might* have changed. + + This should normally be overridden; default will probably capture + creation/modification timestamp (which would be good) or maybe + access timestamp (which would be bad) + """ + return int(tokenize(self.info(path)), 16) + + def size(self, path): + """Size in bytes of file""" + return self.info(path).get("size", None) + + def sizes(self, paths): + """Size in bytes of each file in a list of paths""" + return [self.size(p) for p in paths] + + def isdir(self, path): + """Is this entry directory-like?""" + try: + return self.info(path)["type"] == "directory" + except OSError: + return False + + def isfile(self, path): + """Is this entry file-like?""" + try: + return self.info(path)["type"] == "file" + except: # noqa: E722 + return False + + def read_text(self, path, encoding=None, errors=None, newline=None, **kwargs): + """Get the contents of the file as a string. + + Parameters + ---------- + path: str + URL of file on this filesystems + encoding, errors, newline: same as `open`. + """ + with self.open( + path, + mode="r", + encoding=encoding, + errors=errors, + newline=newline, + **kwargs, + ) as f: + return f.read() + + def write_text( + self, path, value, encoding=None, errors=None, newline=None, **kwargs + ): + """Write the text to the given file. + + An existing file will be overwritten. + + Parameters + ---------- + path: str + URL of file on this filesystems + value: str + Text to write. + encoding, errors, newline: same as `open`. + """ + with self.open( + path, + mode="w", + encoding=encoding, + errors=errors, + newline=newline, + **kwargs, + ) as f: + return f.write(value) + + def cat_file(self, path, start=None, end=None, **kwargs): + """Get the content of a file + + Parameters + ---------- + path: URL of file on this filesystems + start, end: int + Bytes limits of the read. If negative, backwards from end, + like usual python slices. Either can be None for start or + end of file, respectively + kwargs: passed to ``open()``. + """ + # explicitly set buffering off? + with self.open(path, "rb", **kwargs) as f: + if start is not None: + if start >= 0: + f.seek(start) + else: + f.seek(max(0, f.size + start)) + if end is not None: + if end < 0: + end = f.size + end + return f.read(end - f.tell()) + return f.read() + + def pipe_file(self, path, value, mode="overwrite", **kwargs): + """Set the bytes of given file""" + if mode == "create" and self.exists(path): + # non-atomic but simple way; or could use "xb" in open(), which is likely + # not as well supported + raise FileExistsError + with self.open(path, "wb", **kwargs) as f: + f.write(value) + + def pipe(self, path, value=None, **kwargs): + """Put value into path + + (counterpart to ``cat``) + + Parameters + ---------- + path: string or dict(str, bytes) + If a string, a single remote location to put ``value`` bytes; if a dict, + a mapping of {path: bytesvalue}. + value: bytes, optional + If using a single path, these are the bytes to put there. Ignored if + ``path`` is a dict + """ + if isinstance(path, str): + self.pipe_file(self._strip_protocol(path), value, **kwargs) + elif isinstance(path, dict): + for k, v in path.items(): + self.pipe_file(self._strip_protocol(k), v, **kwargs) + else: + raise ValueError("path must be str or dict") + + def cat_ranges( + self, paths, starts, ends, max_gap=None, on_error="return", **kwargs + ): + """Get the contents of byte ranges from one or more files + + Parameters + ---------- + paths: list + A list of of filepaths on this filesystems + starts, ends: int or list + Bytes limits of the read. If using a single int, the same value will be + used to read all the specified files. + """ + if max_gap is not None: + raise NotImplementedError + if not isinstance(paths, list): + raise TypeError + if not isinstance(starts, list): + starts = [starts] * len(paths) + if not isinstance(ends, list): + ends = [ends] * len(paths) + if len(starts) != len(paths) or len(ends) != len(paths): + raise ValueError + out = [] + for p, s, e in zip(paths, starts, ends): + try: + out.append(self.cat_file(p, s, e)) + except Exception as e: + if on_error == "return": + out.append(e) + else: + raise + return out + + def cat(self, path, recursive=False, on_error="raise", **kwargs): + """Fetch (potentially multiple) paths' contents + + Parameters + ---------- + recursive: bool + If True, assume the path(s) are directories, and get all the + contained files + on_error : "raise", "omit", "return" + If raise, an underlying exception will be raised (converted to KeyError + if the type is in self.missing_exceptions); if omit, keys with exception + will simply not be included in the output; if "return", all keys are + included in the output, but the value will be bytes or an exception + instance. + kwargs: passed to cat_file + + Returns + ------- + dict of {path: contents} if there are multiple paths + or the path has been otherwise expanded + """ + paths = self.expand_path(path, recursive=recursive) + if ( + len(paths) > 1 + or isinstance(path, list) + or paths[0] != self._strip_protocol(path) + ): + out = {} + for path in paths: + try: + out[path] = self.cat_file(path, **kwargs) + except Exception as e: + if on_error == "raise": + raise + if on_error == "return": + out[path] = e + return out + else: + return self.cat_file(paths[0], **kwargs) + + def get_file(self, rpath, lpath, callback=DEFAULT_CALLBACK, outfile=None, **kwargs): + """Copy single remote file to local""" + from .implementations.local import LocalFileSystem + + if isfilelike(lpath): + outfile = lpath + elif self.isdir(rpath): + os.makedirs(lpath, exist_ok=True) + return None + + fs = LocalFileSystem(auto_mkdir=True) + fs.makedirs(fs._parent(lpath), exist_ok=True) + + with self.open(rpath, "rb", **kwargs) as f1: + if outfile is None: + outfile = open(lpath, "wb") + + try: + callback.set_size(getattr(f1, "size", None)) + data = True + while data: + data = f1.read(self.blocksize) + segment_len = outfile.write(data) + if segment_len is None: + segment_len = len(data) + callback.relative_update(segment_len) + finally: + if not isfilelike(lpath): + outfile.close() + + def get( + self, + rpath, + lpath, + recursive=False, + callback=DEFAULT_CALLBACK, + maxdepth=None, + **kwargs, + ): + """Copy file(s) to local. + + Copies a specific file or tree of files (if recursive=True). If lpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. Can submit a list of paths, which may be glob-patterns + and will be expanded. + + Calls get_file for each source. + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + from .implementations.local import ( + LocalFileSystem, + make_path_posix, + trailing_sep, + ) + + source_is_str = isinstance(rpath, str) + rpaths = self.expand_path(rpath, recursive=recursive, maxdepth=maxdepth) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + rpaths = [p for p in rpaths if not (trailing_sep(p) or self.isdir(p))] + if not rpaths: + return + + if isinstance(lpath, str): + lpath = make_path_posix(lpath) + + source_is_file = len(rpaths) == 1 + dest_is_dir = isinstance(lpath, str) and ( + trailing_sep(lpath) or LocalFileSystem().isdir(lpath) + ) + + exists = source_is_str and ( + (has_magic(rpath) and source_is_file) + or (not has_magic(rpath) and dest_is_dir and not trailing_sep(rpath)) + ) + lpaths = other_paths( + rpaths, + lpath, + exists=exists, + flatten=not source_is_str, + ) + + callback.set_size(len(lpaths)) + for lpath, rpath in callback.wrap(zip(lpaths, rpaths)): + with callback.branched(rpath, lpath) as child: + self.get_file(rpath, lpath, callback=child, **kwargs) + + def put_file( + self, lpath, rpath, callback=DEFAULT_CALLBACK, mode="overwrite", **kwargs + ): + """Copy single file to remote""" + if mode == "create" and self.exists(rpath): + raise FileExistsError + if os.path.isdir(lpath): + self.makedirs(rpath, exist_ok=True) + return None + + with open(lpath, "rb") as f1: + size = f1.seek(0, 2) + callback.set_size(size) + f1.seek(0) + + self.mkdirs(self._parent(os.fspath(rpath)), exist_ok=True) + with self.open(rpath, "wb", **kwargs) as f2: + while f1.tell() < size: + data = f1.read(self.blocksize) + segment_len = f2.write(data) + if segment_len is None: + segment_len = len(data) + callback.relative_update(segment_len) + + def put( + self, + lpath, + rpath, + recursive=False, + callback=DEFAULT_CALLBACK, + maxdepth=None, + **kwargs, + ): + """Copy file(s) from local. + + Copies a specific file or tree of files (if recursive=True). If rpath + ends with a "/", it will be assumed to be a directory, and target files + will go within. + + Calls put_file for each source. + """ + if isinstance(lpath, list) and isinstance(rpath, list): + # No need to expand paths when both source and destination + # are provided as lists + rpaths = rpath + lpaths = lpath + else: + from .implementations.local import ( + LocalFileSystem, + make_path_posix, + trailing_sep, + ) + + source_is_str = isinstance(lpath, str) + if source_is_str: + lpath = make_path_posix(lpath) + fs = LocalFileSystem() + lpaths = fs.expand_path(lpath, recursive=recursive, maxdepth=maxdepth) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + lpaths = [p for p in lpaths if not (trailing_sep(p) or fs.isdir(p))] + if not lpaths: + return + + source_is_file = len(lpaths) == 1 + dest_is_dir = isinstance(rpath, str) and ( + trailing_sep(rpath) or self.isdir(rpath) + ) + + rpath = ( + self._strip_protocol(rpath) + if isinstance(rpath, str) + else [self._strip_protocol(p) for p in rpath] + ) + exists = source_is_str and ( + (has_magic(lpath) and source_is_file) + or (not has_magic(lpath) and dest_is_dir and not trailing_sep(lpath)) + ) + rpaths = other_paths( + lpaths, + rpath, + exists=exists, + flatten=not source_is_str, + ) + + callback.set_size(len(rpaths)) + for lpath, rpath in callback.wrap(zip(lpaths, rpaths)): + with callback.branched(lpath, rpath) as child: + self.put_file(lpath, rpath, callback=child, **kwargs) + + def head(self, path, size=1024): + """Get the first ``size`` bytes from file""" + with self.open(path, "rb") as f: + return f.read(size) + + def tail(self, path, size=1024): + """Get the last ``size`` bytes from file""" + with self.open(path, "rb") as f: + f.seek(max(-size, -f.size), 2) + return f.read() + + def cp_file(self, path1, path2, **kwargs): + raise NotImplementedError + + def copy( + self, path1, path2, recursive=False, maxdepth=None, on_error=None, **kwargs + ): + """Copy within two locations in the filesystem + + on_error : "raise", "ignore" + If raise, any not-found exceptions will be raised; if ignore any + not-found exceptions will cause the path to be skipped; defaults to + raise unless recursive is true, where the default is ignore + """ + if on_error is None and recursive: + on_error = "ignore" + elif on_error is None: + on_error = "raise" + + if isinstance(path1, list) and isinstance(path2, list): + # No need to expand paths when both source and destination + # are provided as lists + paths1 = path1 + paths2 = path2 + else: + from .implementations.local import trailing_sep + + source_is_str = isinstance(path1, str) + paths1 = self.expand_path(path1, recursive=recursive, maxdepth=maxdepth) + if source_is_str and (not recursive or maxdepth is not None): + # Non-recursive glob does not copy directories + paths1 = [p for p in paths1 if not (trailing_sep(p) or self.isdir(p))] + if not paths1: + return + + source_is_file = len(paths1) == 1 + dest_is_dir = isinstance(path2, str) and ( + trailing_sep(path2) or self.isdir(path2) + ) + + exists = source_is_str and ( + (has_magic(path1) and source_is_file) + or (not has_magic(path1) and dest_is_dir and not trailing_sep(path1)) + ) + paths2 = other_paths( + paths1, + path2, + exists=exists, + flatten=not source_is_str, + ) + + for p1, p2 in zip(paths1, paths2): + try: + self.cp_file(p1, p2, **kwargs) + except FileNotFoundError: + if on_error == "raise": + raise + + def expand_path(self, path, recursive=False, maxdepth=None, **kwargs): + """Turn one or more globs or directories into a list of all matching paths + to files or directories. + + kwargs are passed to ``glob`` or ``find``, which may in turn call ``ls`` + """ + + if maxdepth is not None and maxdepth < 1: + raise ValueError("maxdepth must be at least 1") + + if isinstance(path, (str, os.PathLike)): + out = self.expand_path([path], recursive, maxdepth) + else: + out = set() + path = [self._strip_protocol(p) for p in path] + for p in path: + if has_magic(p): + bit = set(self.glob(p, maxdepth=maxdepth, **kwargs)) + out |= bit + if recursive: + # glob call above expanded one depth so if maxdepth is defined + # then decrement it in expand_path call below. If it is zero + # after decrementing then avoid expand_path call. + if maxdepth is not None and maxdepth <= 1: + continue + out |= set( + self.expand_path( + list(bit), + recursive=recursive, + maxdepth=maxdepth - 1 if maxdepth is not None else None, + **kwargs, + ) + ) + continue + elif recursive: + rec = set( + self.find( + p, maxdepth=maxdepth, withdirs=True, detail=False, **kwargs + ) + ) + out |= rec + if p not in out and (recursive is False or self.exists(p)): + # should only check once, for the root + out.add(p) + if not out: + raise FileNotFoundError(path) + return sorted(out) + + def mv(self, path1, path2, recursive=False, maxdepth=None, **kwargs): + """Move file(s) from one location to another""" + if path1 == path2: + logger.debug("%s mv: The paths are the same, so no files were moved.", self) + else: + # explicitly raise exception to prevent data corruption + self.copy( + path1, path2, recursive=recursive, maxdepth=maxdepth, onerror="raise" + ) + self.rm(path1, recursive=recursive) + + def rm_file(self, path): + """Delete a file""" + self._rm(path) + + def _rm(self, path): + """Delete one file""" + # this is the old name for the method, prefer rm_file + raise NotImplementedError + + def rm(self, path, recursive=False, maxdepth=None): + """Delete files. + + Parameters + ---------- + path: str or list of str + File(s) to delete. + recursive: bool + If file(s) are directories, recursively delete contents and then + also remove the directory + maxdepth: int or None + Depth to pass to walk for finding files to delete, if recursive. + If None, there will be no limit and infinite recursion may be + possible. + """ + path = self.expand_path(path, recursive=recursive, maxdepth=maxdepth) + for p in reversed(path): + self.rm_file(p) + + @classmethod + def _parent(cls, path): + path = cls._strip_protocol(path) + if "/" in path: + parent = path.rsplit("/", 1)[0].lstrip(cls.root_marker) + return cls.root_marker + parent + else: + return cls.root_marker + + def _open( + self, + path, + mode="rb", + block_size=None, + autocommit=True, + cache_options=None, + **kwargs, + ): + """Return raw bytes-mode file-like from the file-system""" + return AbstractBufferedFile( + self, + path, + mode, + block_size, + autocommit, + cache_options=cache_options, + **kwargs, + ) + + def open( + self, + path, + mode="rb", + block_size=None, + cache_options=None, + compression=None, + **kwargs, + ): + """ + Return a file-like object from the filesystem + + The resultant instance must function correctly in a context ``with`` + block. + + Parameters + ---------- + path: str + Target file + mode: str like 'rb', 'w' + See builtin ``open()`` + Mode "x" (exclusive write) may be implemented by the backend. Even if + it is, whether it is checked up front or on commit, and whether it is + atomic is implementation-dependent. + block_size: int + Some indication of buffering - this is a value in bytes + cache_options : dict, optional + Extra arguments to pass through to the cache. + compression: string or None + If given, open file using compression codec. Can either be a compression + name (a key in ``fsspec.compression.compr``) or "infer" to guess the + compression from the filename suffix. + encoding, errors, newline: passed on to TextIOWrapper for text mode + """ + import io + + path = self._strip_protocol(path) + if "b" not in mode: + mode = mode.replace("t", "") + "b" + + text_kwargs = { + k: kwargs.pop(k) + for k in ["encoding", "errors", "newline"] + if k in kwargs + } + return io.TextIOWrapper( + self.open( + path, + mode, + block_size=block_size, + cache_options=cache_options, + compression=compression, + **kwargs, + ), + **text_kwargs, + ) + else: + ac = kwargs.pop("autocommit", not self._intrans) + f = self._open( + path, + mode=mode, + block_size=block_size, + autocommit=ac, + cache_options=cache_options, + **kwargs, + ) + if compression is not None: + from fsspec.compression import compr + from fsspec.core import get_compression + + compression = get_compression(path, compression) + compress = compr[compression] + f = compress(f, mode=mode[0]) + + if not ac and "r" not in mode: + self.transaction.files.append(f) + return f + + def touch(self, path, truncate=True, **kwargs): + """Create empty file, or update timestamp + + Parameters + ---------- + path: str + file location + truncate: bool + If True, always set file size to 0; if False, update timestamp and + leave file unchanged, if backend allows this + """ + if truncate or not self.exists(path): + with self.open(path, "wb", **kwargs): + pass + else: + raise NotImplementedError # update timestamp, if possible + + def ukey(self, path): + """Hash of file properties, to tell if it has changed""" + return sha256(str(self.info(path)).encode()).hexdigest() + + def read_block(self, fn, offset, length, delimiter=None): + """Read a block of bytes from + + Starting at ``offset`` of the file, read ``length`` bytes. If + ``delimiter`` is set then we ensure that the read starts and stops at + delimiter boundaries that follow the locations ``offset`` and ``offset + + length``. If ``offset`` is zero then we start at zero. The + bytestring returned WILL include the end delimiter string. + + If offset+length is beyond the eof, reads to eof. + + Parameters + ---------- + fn: string + Path to filename + offset: int + Byte offset to start read + length: int + Number of bytes to read. If None, read to end. + delimiter: bytes (optional) + Ensure reading starts and stops at delimiter bytestring + + Examples + -------- + >>> fs.read_block('data/file.csv', 0, 13) # doctest: +SKIP + b'Alice, 100\\nBo' + >>> fs.read_block('data/file.csv', 0, 13, delimiter=b'\\n') # doctest: +SKIP + b'Alice, 100\\nBob, 200\\n' + + Use ``length=None`` to read to the end of the file. + >>> fs.read_block('data/file.csv', 0, None, delimiter=b'\\n') # doctest: +SKIP + b'Alice, 100\\nBob, 200\\nCharlie, 300' + + See Also + -------- + :func:`fsspec.utils.read_block` + """ + with self.open(fn, "rb") as f: + size = f.size + if length is None: + length = size + if size is not None and offset + length > size: + length = size - offset + return read_block(f, offset, length, delimiter) + + def to_json(self, *, include_password: bool = True) -> str: + """ + JSON representation of this filesystem instance. + + Parameters + ---------- + include_password: bool, default True + Whether to include the password (if any) in the output. + + Returns + ------- + JSON string with keys ``cls`` (the python location of this class), + protocol (text name of this class's protocol, first one in case of + multiple), ``args`` (positional args, usually empty), and all other + keyword arguments as their own keys. + + Warnings + -------- + Serialized filesystems may contain sensitive information which have been + passed to the constructor, such as passwords and tokens. Make sure you + store and send them in a secure environment! + """ + from .json import FilesystemJSONEncoder + + return json.dumps( + self, + cls=type( + "_FilesystemJSONEncoder", + (FilesystemJSONEncoder,), + {"include_password": include_password}, + ), + ) + + @staticmethod + def from_json(blob: str) -> AbstractFileSystem: + """ + Recreate a filesystem instance from JSON representation. + + See ``.to_json()`` for the expected structure of the input. + + Parameters + ---------- + blob: str + + Returns + ------- + file system instance, not necessarily of this particular class. + + Warnings + -------- + This can import arbitrary modules (as determined by the ``cls`` key). + Make sure you haven't installed any modules that may execute malicious code + at import time. + """ + from .json import FilesystemJSONDecoder + + return json.loads(blob, cls=FilesystemJSONDecoder) + + def to_dict(self, *, include_password: bool = True) -> dict[str, Any]: + """ + JSON-serializable dictionary representation of this filesystem instance. + + Parameters + ---------- + include_password: bool, default True + Whether to include the password (if any) in the output. + + Returns + ------- + Dictionary with keys ``cls`` (the python location of this class), + protocol (text name of this class's protocol, first one in case of + multiple), ``args`` (positional args, usually empty), and all other + keyword arguments as their own keys. + + Warnings + -------- + Serialized filesystems may contain sensitive information which have been + passed to the constructor, such as passwords and tokens. Make sure you + store and send them in a secure environment! + """ + from .json import FilesystemJSONEncoder + + json_encoder = FilesystemJSONEncoder() + + cls = type(self) + proto = self.protocol + + storage_options = dict(self.storage_options) + if not include_password: + storage_options.pop("password", None) + + return dict( + cls=f"{cls.__module__}:{cls.__name__}", + protocol=proto[0] if isinstance(proto, (tuple, list)) else proto, + args=json_encoder.make_serializable(self.storage_args), + **json_encoder.make_serializable(storage_options), + ) + + @staticmethod + def from_dict(dct: dict[str, Any]) -> AbstractFileSystem: + """ + Recreate a filesystem instance from dictionary representation. + + See ``.to_dict()`` for the expected structure of the input. + + Parameters + ---------- + dct: Dict[str, Any] + + Returns + ------- + file system instance, not necessarily of this particular class. + + Warnings + -------- + This can import arbitrary modules (as determined by the ``cls`` key). + Make sure you haven't installed any modules that may execute malicious code + at import time. + """ + from .json import FilesystemJSONDecoder + + json_decoder = FilesystemJSONDecoder() + + dct = dict(dct) # Defensive copy + + cls = FilesystemJSONDecoder.try_resolve_fs_cls(dct) + if cls is None: + raise ValueError("Not a serialized AbstractFileSystem") + + dct.pop("cls", None) + dct.pop("protocol", None) + + return cls( + *json_decoder.unmake_serializable(dct.pop("args", ())), + **json_decoder.unmake_serializable(dct), + ) + + def _get_pyarrow_filesystem(self): + """ + Make a version of the FS instance which will be acceptable to pyarrow + """ + # all instances already also derive from pyarrow + return self + + def get_mapper(self, root="", check=False, create=False, missing_exceptions=None): + """Create key/value store based on this file-system + + Makes a MutableMapping interface to the FS at the given root path. + See ``fsspec.mapping.FSMap`` for further details. + """ + from .mapping import FSMap + + return FSMap( + root, + self, + check=check, + create=create, + missing_exceptions=missing_exceptions, + ) + + @classmethod + def clear_instance_cache(cls): + """ + Clear the cache of filesystem instances. + + Notes + ----- + Unless overridden by setting the ``cachable`` class attribute to False, + the filesystem class stores a reference to newly created instances. This + prevents Python's normal rules around garbage collection from working, + since the instances refcount will not drop to zero until + ``clear_instance_cache`` is called. + """ + cls._cache.clear() + + def created(self, path): + """Return the created timestamp of a file as a datetime.datetime""" + raise NotImplementedError + + def modified(self, path): + """Return the modified timestamp of a file as a datetime.datetime""" + raise NotImplementedError + + def tree( + self, + path: str = "/", + recursion_limit: int = 2, + max_display: int = 25, + display_size: bool = False, + prefix: str = "", + is_last: bool = True, + first: bool = True, + indent_size: int = 4, + ) -> str: + """ + Return a tree-like structure of the filesystem starting from the given path as a string. + + Parameters + ---------- + path: Root path to start traversal from + recursion_limit: Maximum depth of directory traversal + max_display: Maximum number of items to display per directory + display_size: Whether to display file sizes + prefix: Current line prefix for visual tree structure + is_last: Whether current item is last in its level + first: Whether this is the first call (displays root path) + indent_size: Number of spaces by indent + + Returns + ------- + str: A string representing the tree structure. + + Example + ------- + >>> from fsspec import filesystem + + >>> fs = filesystem('ftp', host='test.rebex.net', user='demo', password='password') + >>> tree = fs.tree(display_size=True, recursion_limit=3, indent_size=8, max_display=10) + >>> print(tree) + """ + + def format_bytes(n: int) -> str: + """Format bytes as text.""" + for prefix, k in ( + ("P", 2**50), + ("T", 2**40), + ("G", 2**30), + ("M", 2**20), + ("k", 2**10), + ): + if n >= 0.9 * k: + return f"{n / k:.2f} {prefix}b" + return f"{n}B" + + result = [] + + if first: + result.append(path) + + if recursion_limit: + indent = " " * indent_size + contents = self.ls(path, detail=True) + contents.sort( + key=lambda x: (x.get("type") != "directory", x.get("name", "")) + ) + + if max_display is not None and len(contents) > max_display: + displayed_contents = contents[:max_display] + remaining_count = len(contents) - max_display + else: + displayed_contents = contents + remaining_count = 0 + + for i, item in enumerate(displayed_contents): + is_last_item = (i == len(displayed_contents) - 1) and ( + remaining_count == 0 + ) + + branch = ( + "└" + ("─" * (indent_size - 2)) + if is_last_item + else "├" + ("─" * (indent_size - 2)) + ) + branch += " " + new_prefix = prefix + ( + indent if is_last_item else "│" + " " * (indent_size - 1) + ) + + name = os.path.basename(item.get("name", "")) + + if display_size and item.get("type") == "directory": + sub_contents = self.ls(item.get("name", ""), detail=True) + num_files = sum( + 1 for sub_item in sub_contents if sub_item.get("type") == "file" + ) + num_folders = sum( + 1 + for sub_item in sub_contents + if sub_item.get("type") == "directory" + ) + + if num_files == 0 and num_folders == 0: + size = " (empty folder)" + elif num_files == 0: + size = f" ({num_folders} subfolder{'s' if num_folders > 1 else ''})" + elif num_folders == 0: + size = f" ({num_files} file{'s' if num_files > 1 else ''})" + else: + size = f" ({num_files} file{'s' if num_files > 1 else ''}, {num_folders} subfolder{'s' if num_folders > 1 else ''})" + elif display_size and item.get("type") == "file": + size = f" ({format_bytes(item.get('size', 0))})" + else: + size = "" + + result.append(f"{prefix}{branch}{name}{size}") + + if item.get("type") == "directory" and recursion_limit > 0: + result.append( + self.tree( + path=item.get("name", ""), + recursion_limit=recursion_limit - 1, + max_display=max_display, + display_size=display_size, + prefix=new_prefix, + is_last=is_last_item, + first=False, + indent_size=indent_size, + ) + ) + + if remaining_count > 0: + more_message = f"{remaining_count} more item(s) not displayed." + result.append( + f"{prefix}{'└' + ('─' * (indent_size - 2))} {more_message}" + ) + + return "\n".join(_ for _ in result if _) + + # ------------------------------------------------------------------------ + # Aliases + + def read_bytes(self, path, start=None, end=None, **kwargs): + """Alias of `AbstractFileSystem.cat_file`.""" + return self.cat_file(path, start=start, end=end, **kwargs) + + def write_bytes(self, path, value, **kwargs): + """Alias of `AbstractFileSystem.pipe_file`.""" + self.pipe_file(path, value, **kwargs) + + def makedir(self, path, create_parents=True, **kwargs): + """Alias of `AbstractFileSystem.mkdir`.""" + return self.mkdir(path, create_parents=create_parents, **kwargs) + + def mkdirs(self, path, exist_ok=False): + """Alias of `AbstractFileSystem.makedirs`.""" + return self.makedirs(path, exist_ok=exist_ok) + + def listdir(self, path, detail=True, **kwargs): + """Alias of `AbstractFileSystem.ls`.""" + return self.ls(path, detail=detail, **kwargs) + + def cp(self, path1, path2, **kwargs): + """Alias of `AbstractFileSystem.copy`.""" + return self.copy(path1, path2, **kwargs) + + def move(self, path1, path2, **kwargs): + """Alias of `AbstractFileSystem.mv`.""" + return self.mv(path1, path2, **kwargs) + + def stat(self, path, **kwargs): + """Alias of `AbstractFileSystem.info`.""" + return self.info(path, **kwargs) + + def disk_usage(self, path, total=True, maxdepth=None, **kwargs): + """Alias of `AbstractFileSystem.du`.""" + return self.du(path, total=total, maxdepth=maxdepth, **kwargs) + + def rename(self, path1, path2, **kwargs): + """Alias of `AbstractFileSystem.mv`.""" + return self.mv(path1, path2, **kwargs) + + def delete(self, path, recursive=False, maxdepth=None): + """Alias of `AbstractFileSystem.rm`.""" + return self.rm(path, recursive=recursive, maxdepth=maxdepth) + + def upload(self, lpath, rpath, recursive=False, **kwargs): + """Alias of `AbstractFileSystem.put`.""" + return self.put(lpath, rpath, recursive=recursive, **kwargs) + + def download(self, rpath, lpath, recursive=False, **kwargs): + """Alias of `AbstractFileSystem.get`.""" + return self.get(rpath, lpath, recursive=recursive, **kwargs) + + def sign(self, path, expiration=100, **kwargs): + """Create a signed URL representing the given path + + Some implementations allow temporary URLs to be generated, as a + way of delegating credentials. + + Parameters + ---------- + path : str + The path on the filesystem + expiration : int + Number of seconds to enable the URL for (if supported) + + Returns + ------- + URL : str + The signed URL + + Raises + ------ + NotImplementedError : if method is not implemented for a filesystem + """ + raise NotImplementedError("Sign is not implemented for this filesystem") + + def _isfilestore(self): + # Originally inherited from pyarrow DaskFileSystem. Keeping this + # here for backwards compatibility as long as pyarrow uses its + # legacy fsspec-compatible filesystems and thus accepts fsspec + # filesystems as well + return False + + +class AbstractBufferedFile(io.IOBase): + """Convenient class to derive from to provide buffering + + In the case that the backend does not provide a pythonic file-like object + already, this class contains much of the logic to build one. The only + methods that need to be overridden are ``_upload_chunk``, + ``_initiate_upload`` and ``_fetch_range``. + """ + + DEFAULT_BLOCK_SIZE = 5 * 2**20 + _details = None + + def __init__( + self, + fs, + path, + mode="rb", + block_size="default", + autocommit=True, + cache_type="readahead", + cache_options=None, + size=None, + **kwargs, + ): + """ + Template for files with buffered reading and writing + + Parameters + ---------- + fs: instance of FileSystem + path: str + location in file-system + mode: str + Normal file modes. Currently only 'wb', 'ab' or 'rb'. Some file + systems may be read-only, and some may not support append. + block_size: int + Buffer size for reading or writing, 'default' for class default + autocommit: bool + Whether to write to final destination; may only impact what + happens when file is being closed. + cache_type: {"readahead", "none", "mmap", "bytes"}, default "readahead" + Caching policy in read mode. See the definitions in ``core``. + cache_options : dict + Additional options passed to the constructor for the cache specified + by `cache_type`. + size: int + If given and in read mode, suppressed having to look up the file size + kwargs: + Gets stored as self.kwargs + """ + from .core import caches + + self.path = path + self.fs = fs + self.mode = mode + self.blocksize = ( + self.DEFAULT_BLOCK_SIZE if block_size in ["default", None] else block_size + ) + self.loc = 0 + self.autocommit = autocommit + self.end = None + self.start = None + self.closed = False + + if cache_options is None: + cache_options = {} + + if "trim" in kwargs: + warnings.warn( + "Passing 'trim' to control the cache behavior has been deprecated. " + "Specify it within the 'cache_options' argument instead.", + FutureWarning, + ) + cache_options["trim"] = kwargs.pop("trim") + + self.kwargs = kwargs + + if mode not in {"ab", "rb", "wb", "xb"}: + raise NotImplementedError("File mode not supported") + if mode == "rb": + if size is not None: + self.size = size + else: + self.size = self.details["size"] + self.cache = caches[cache_type]( + self.blocksize, self._fetch_range, self.size, **cache_options + ) + else: + self.buffer = io.BytesIO() + self.offset = None + self.forced = False + self.location = None + + @property + def details(self): + if self._details is None: + self._details = self.fs.info(self.path) + return self._details + + @details.setter + def details(self, value): + self._details = value + self.size = value["size"] + + @property + def full_name(self): + return _unstrip_protocol(self.path, self.fs) + + @property + def closed(self): + # get around this attr being read-only in IOBase + # use getattr here, since this can be called during del + return getattr(self, "_closed", True) + + @closed.setter + def closed(self, c): + self._closed = c + + def __hash__(self): + if "w" in self.mode: + return id(self) + else: + return int(tokenize(self.details), 16) + + def __eq__(self, other): + """Files are equal if they have the same checksum, only in read mode""" + if self is other: + return True + return ( + isinstance(other, type(self)) + and self.mode == "rb" + and other.mode == "rb" + and hash(self) == hash(other) + ) + + def commit(self): + """Move from temp to final destination""" + + def discard(self): + """Throw away temporary file""" + + def info(self): + """File information about this path""" + if self.readable(): + return self.details + else: + raise ValueError("Info not available while writing") + + def tell(self): + """Current file location""" + return self.loc + + def seek(self, loc, whence=0): + """Set current file location + + Parameters + ---------- + loc: int + byte location + whence: {0, 1, 2} + from start of file, current location or end of file, resp. + """ + loc = int(loc) + if not self.mode == "rb": + raise OSError(ESPIPE, "Seek only available in read mode") + if whence == 0: + nloc = loc + elif whence == 1: + nloc = self.loc + loc + elif whence == 2: + nloc = self.size + loc + else: + raise ValueError(f"invalid whence ({whence}, should be 0, 1 or 2)") + if nloc < 0: + raise ValueError("Seek before start of file") + self.loc = nloc + return self.loc + + def write(self, data): + """ + Write data to buffer. + + Buffer only sent on flush() or if buffer is greater than + or equal to blocksize. + + Parameters + ---------- + data: bytes + Set of bytes to be written. + """ + if not self.writable(): + raise ValueError("File not in write mode") + if self.closed: + raise ValueError("I/O operation on closed file.") + if self.forced: + raise ValueError("This file has been force-flushed, can only close") + out = self.buffer.write(data) + self.loc += out + if self.buffer.tell() >= self.blocksize: + self.flush() + return out + + def flush(self, force=False): + """ + Write buffered data to backend store. + + Writes the current buffer, if it is larger than the block-size, or if + the file is being closed. + + Parameters + ---------- + force: bool + When closing, write the last block even if it is smaller than + blocks are allowed to be. Disallows further writing to this file. + """ + + if self.closed: + raise ValueError("Flush on closed file") + if force and self.forced: + raise ValueError("Force flush cannot be called more than once") + if force: + self.forced = True + + if self.readable(): + # no-op to flush on read-mode + return + + if not force and self.buffer.tell() < self.blocksize: + # Defer write on small block + return + + if self.offset is None: + # Initialize a multipart upload + self.offset = 0 + try: + self._initiate_upload() + except: + self.closed = True + raise + + if self._upload_chunk(final=force) is not False: + self.offset += self.buffer.seek(0, 2) + self.buffer = io.BytesIO() + + def _upload_chunk(self, final=False): + """Write one part of a multi-block file upload + + Parameters + ========== + final: bool + This is the last block, so should complete file, if + self.autocommit is True. + """ + # may not yet have been initialized, may need to call _initialize_upload + + def _initiate_upload(self): + """Create remote file/upload""" + pass + + def _fetch_range(self, start, end): + """Get the specified set of bytes from remote""" + return self.fs.cat_file(self.path, start=start, end=end) + + def read(self, length=-1): + """ + Return data from cache, or fetch pieces as necessary + + Parameters + ---------- + length: int (-1) + Number of bytes to read; if <0, all remaining bytes. + """ + length = -1 if length is None else int(length) + if self.mode != "rb": + raise ValueError("File not in read mode") + if length < 0: + length = self.size - self.loc + if self.closed: + raise ValueError("I/O operation on closed file.") + if length == 0: + # don't even bother calling fetch + return b"" + out = self.cache._fetch(self.loc, self.loc + length) + + logger.debug( + "%s read: %i - %i %s", + self, + self.loc, + self.loc + length, + self.cache._log_stats(), + ) + self.loc += len(out) + return out + + def readinto(self, b): + """mirrors builtin file's readinto method + + https://docs.python.org/3/library/io.html#io.RawIOBase.readinto + """ + out = memoryview(b).cast("B") + data = self.read(out.nbytes) + out[: len(data)] = data + return len(data) + + def readuntil(self, char=b"\n", blocks=None): + """Return data between current position and first occurrence of char + + char is included in the output, except if the end of the tile is + encountered first. + + Parameters + ---------- + char: bytes + Thing to find + blocks: None or int + How much to read in each go. Defaults to file blocksize - which may + mean a new read on every call. + """ + out = [] + while True: + start = self.tell() + part = self.read(blocks or self.blocksize) + if len(part) == 0: + break + found = part.find(char) + if found > -1: + out.append(part[: found + len(char)]) + self.seek(start + found + len(char)) + break + out.append(part) + return b"".join(out) + + def readline(self): + """Read until and including the first occurrence of newline character + + Note that, because of character encoding, this is not necessarily a + true line ending. + """ + return self.readuntil(b"\n") + + def __next__(self): + out = self.readline() + if out: + return out + raise StopIteration + + def __iter__(self): + return self + + def readlines(self): + """Return all data, split by the newline character, including the newline character""" + data = self.read() + lines = data.split(b"\n") + out = [l + b"\n" for l in lines[:-1]] + if data.endswith(b"\n"): + return out + else: + return out + [lines[-1]] + # return list(self) ??? + + def readinto1(self, b): + return self.readinto(b) + + def close(self): + """Close file + + Finalizes writes, discards cache + """ + if getattr(self, "_unclosable", False): + return + if self.closed: + return + try: + if self.mode == "rb": + self.cache = None + else: + if not self.forced: + self.flush(force=True) + + if self.fs is not None: + self.fs.invalidate_cache(self.path) + self.fs.invalidate_cache(self.fs._parent(self.path)) + finally: + self.closed = True + + def readable(self): + """Whether opened for reading""" + return "r" in self.mode and not self.closed + + def seekable(self): + """Whether is seekable (only in read mode)""" + return self.readable() + + def writable(self): + """Whether opened for writing""" + return self.mode in {"wb", "ab", "xb"} and not self.closed + + def __reduce__(self): + if self.mode != "rb": + raise RuntimeError("Pickling a writeable file is not supported") + + return reopen, ( + self.fs, + self.path, + self.mode, + self.blocksize, + self.loc, + self.size, + self.autocommit, + self.cache.name if self.cache else "none", + self.kwargs, + ) + + def __del__(self): + if not self.closed: + self.close() + + def __str__(self): + return f"" + + __repr__ = __str__ + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + +def reopen(fs, path, mode, blocksize, loc, size, autocommit, cache_type, kwargs): + file = fs.open( + path, + mode=mode, + block_size=blocksize, + autocommit=autocommit, + cache_type=cache_type, + size=size, + **kwargs, + ) + if loc > 0: + file.seek(loc) + return file diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__init__.py b/lib/python3.12/site-packages/fsspec/tests/abstract/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..8ed2ad802ecaf021106c25c03112f29e75c7b2f8 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/__init__.py @@ -0,0 +1,289 @@ +import os +from hashlib import md5 + +import pytest + +from fsspec.implementations.local import LocalFileSystem +from fsspec.tests.abstract.copy import AbstractCopyTests # noqa: F401 +from fsspec.tests.abstract.get import AbstractGetTests # noqa: F401 +from fsspec.tests.abstract.open import AbstractOpenTests # noqa: F401 +from fsspec.tests.abstract.pipe import AbstractPipeTests # noqa: F401 +from fsspec.tests.abstract.put import AbstractPutTests # noqa: F401 + + +class BaseAbstractFixtures: + """ + Abstract base class containing fixtures that are used by but never need to + be overridden in derived filesystem-specific classes to run the abstract + tests on such filesystems. + """ + + @pytest.fixture + def fs_bulk_operations_scenario_0(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used for many cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._bulk_operations_scenario_0(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_glob_edge_cases_files(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used for glob edge cases cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._glob_edge_cases_files(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_dir_and_file_with_same_name_prefix(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used to check cp/get/put on directory + and file with the same name prefixes. + + Cleans up at the end of each test it which it is used. + """ + source = self._dir_and_file_with_same_name_prefix(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_10_files_with_hashed_names(self, fs, fs_join, fs_path): + """ + Scenario on remote filesystem that is used to check cp/get/put files order + when source and destination are lists. + + Cleans up at the end of each test it which it is used. + """ + source = self._10_files_with_hashed_names(fs, fs_join, fs_path) + yield source + fs.rm(source, recursive=True) + + @pytest.fixture + def fs_target(self, fs, fs_join, fs_path): + """ + Return name of remote directory that does not yet exist to copy into. + + Cleans up at the end of each test it which it is used. + """ + target = fs_join(fs_path, "target") + yield target + if fs.exists(target): + fs.rm(target, recursive=True) + + @pytest.fixture + def local_bulk_operations_scenario_0(self, local_fs, local_join, local_path): + """ + Scenario on local filesystem that is used for many cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._bulk_operations_scenario_0(local_fs, local_join, local_path) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_glob_edge_cases_files(self, local_fs, local_join, local_path): + """ + Scenario on local filesystem that is used for glob edge cases cp/get/put tests. + + Cleans up at the end of each test it which it is used. + """ + source = self._glob_edge_cases_files(local_fs, local_join, local_path) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_dir_and_file_with_same_name_prefix( + self, local_fs, local_join, local_path + ): + """ + Scenario on local filesystem that is used to check cp/get/put on directory + and file with the same name prefixes. + + Cleans up at the end of each test it which it is used. + """ + source = self._dir_and_file_with_same_name_prefix( + local_fs, local_join, local_path + ) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_10_files_with_hashed_names(self, local_fs, local_join, local_path): + """ + Scenario on local filesystem that is used to check cp/get/put files order + when source and destination are lists. + + Cleans up at the end of each test it which it is used. + """ + source = self._10_files_with_hashed_names(local_fs, local_join, local_path) + yield source + local_fs.rm(source, recursive=True) + + @pytest.fixture + def local_target(self, local_fs, local_join, local_path): + """ + Return name of local directory that does not yet exist to copy into. + + Cleans up at the end of each test it which it is used. + """ + target = local_join(local_path, "target") + yield target + if local_fs.exists(target): + local_fs.rm(target, recursive=True) + + def _glob_edge_cases_files(self, some_fs, some_join, some_path): + """ + Scenario that is used for glob edge cases cp/get/put tests. + Creates the following directory and file structure: + + 📁 source + ├── 📄 file1 + ├── 📄 file2 + ├── 📁 subdir0 + │ ├── 📄 subfile1 + │ ├── 📄 subfile2 + │ └── 📁 nesteddir + │ └── 📄 nestedfile + └── 📁 subdir1 + ├── 📄 subfile1 + ├── 📄 subfile2 + └── 📁 nesteddir + └── 📄 nestedfile + """ + source = some_join(some_path, "source") + some_fs.touch(some_join(source, "file1")) + some_fs.touch(some_join(source, "file2")) + + for subdir_idx in range(2): + subdir = some_join(source, f"subdir{subdir_idx}") + nesteddir = some_join(subdir, "nesteddir") + some_fs.makedirs(nesteddir) + some_fs.touch(some_join(subdir, "subfile1")) + some_fs.touch(some_join(subdir, "subfile2")) + some_fs.touch(some_join(nesteddir, "nestedfile")) + + return source + + def _bulk_operations_scenario_0(self, some_fs, some_join, some_path): + """ + Scenario that is used for many cp/get/put tests. Creates the following + directory and file structure: + + 📁 source + ├── 📄 file1 + ├── 📄 file2 + └── 📁 subdir + ├── 📄 subfile1 + ├── 📄 subfile2 + └── 📁 nesteddir + └── 📄 nestedfile + """ + source = some_join(some_path, "source") + subdir = some_join(source, "subdir") + nesteddir = some_join(subdir, "nesteddir") + some_fs.makedirs(nesteddir) + some_fs.touch(some_join(source, "file1")) + some_fs.touch(some_join(source, "file2")) + some_fs.touch(some_join(subdir, "subfile1")) + some_fs.touch(some_join(subdir, "subfile2")) + some_fs.touch(some_join(nesteddir, "nestedfile")) + return source + + def _dir_and_file_with_same_name_prefix(self, some_fs, some_join, some_path): + """ + Scenario that is used to check cp/get/put on directory and file with + the same name prefixes. Creates the following directory and file structure: + + 📁 source + ├── 📄 subdir.txt + └── 📁 subdir + └── 📄 subfile.txt + """ + source = some_join(some_path, "source") + subdir = some_join(source, "subdir") + file = some_join(source, "subdir.txt") + subfile = some_join(subdir, "subfile.txt") + some_fs.makedirs(subdir) + some_fs.touch(file) + some_fs.touch(subfile) + return source + + def _10_files_with_hashed_names(self, some_fs, some_join, some_path): + """ + Scenario that is used to check cp/get/put files order when source and + destination are lists. Creates the following directory and file structure: + + 📁 source + └── 📄 {hashed([0-9])}.txt + """ + source = some_join(some_path, "source") + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + path = some_join(source, f"{hashed_i}.txt") + some_fs.pipe(path=path, value=f"{i}".encode()) + return source + + +class AbstractFixtures(BaseAbstractFixtures): + """ + Abstract base class containing fixtures that may be overridden in derived + filesystem-specific classes to run the abstract tests on such filesystems. + + For any particular filesystem some of these fixtures must be overridden, + such as ``fs`` and ``fs_path``, and others may be overridden if the + default functions here are not appropriate, such as ``fs_join``. + """ + + @pytest.fixture + def fs(self): + raise NotImplementedError("This function must be overridden in derived classes") + + @pytest.fixture + def fs_join(self): + """ + Return a function that joins its arguments together into a path. + + Most fsspec implementations join paths in a platform-dependent way, + but some will override this to always use a forward slash. + """ + return os.path.join + + @pytest.fixture + def fs_path(self): + raise NotImplementedError("This function must be overridden in derived classes") + + @pytest.fixture(scope="class") + def local_fs(self): + # Maybe need an option for auto_mkdir=False? This is only relevant + # for certain implementations. + return LocalFileSystem(auto_mkdir=True) + + @pytest.fixture + def local_join(self): + """ + Return a function that joins its arguments together into a path, on + the local filesystem. + """ + return os.path.join + + @pytest.fixture + def local_path(self, tmpdir): + return tmpdir + + @pytest.fixture + def supports_empty_directories(self): + """ + Return whether this implementation supports empty directories. + """ + return True + + @pytest.fixture + def fs_sanitize_path(self): + return lambda x: x diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..adfc5bcf7d431c5f2fa6f4d0a46471307812a58b Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/common.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/common.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2c7e4b6d859dd2248c29f69e64569f4934c97eed Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/common.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/copy.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/copy.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..68feafb8a9f204d9e3e923452393692282f4f750 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/copy.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/get.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/get.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e224793dc24989e9b4763030471e988017ba002b Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/get.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/mv.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/mv.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8cab805da86621d1f136c2c2ef819946ca46a41a Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/mv.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/open.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/open.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f7c9b5aee2689db6f63ada7867eccde535d9c48e Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/open.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/pipe.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/pipe.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f2501dce12e897dcc980e9a0dde42c1a2c7cfbb0 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/pipe.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/put.cpython-312.pyc b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/put.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..81de00c0efbf6c548ae66d22a7c06b125b867703 Binary files /dev/null and b/lib/python3.12/site-packages/fsspec/tests/abstract/__pycache__/put.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/common.py b/lib/python3.12/site-packages/fsspec/tests/abstract/common.py new file mode 100644 index 0000000000000000000000000000000000000000..22e7c4140404ab2a8928689721419cf05c2760b9 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/common.py @@ -0,0 +1,175 @@ +GLOB_EDGE_CASES_TESTS = { + "argnames": ("path", "recursive", "maxdepth", "expected"), + "argvalues": [ + ("fil?1", False, None, ["file1"]), + ("fil?1", True, None, ["file1"]), + ("file[1-2]", False, None, ["file1", "file2"]), + ("file[1-2]", True, None, ["file1", "file2"]), + ("*", False, None, ["file1", "file2"]), + ( + "*", + True, + None, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("*", True, 1, ["file1", "file2"]), + ( + "*", + True, + 2, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ("*1", False, None, ["file1"]), + ( + "*1", + True, + None, + [ + "file1", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("*1", True, 2, ["file1", "subdir1/subfile1", "subdir1/subfile2"]), + ( + "**", + False, + None, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ( + "**", + True, + None, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("**", True, 1, ["file1", "file2"]), + ( + "**", + True, + 2, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ( + "**", + False, + 2, + [ + "file1", + "file2", + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ("**/*1", False, None, ["file1", "subdir0/subfile1", "subdir1/subfile1"]), + ( + "**/*1", + True, + None, + [ + "file1", + "subdir0/subfile1", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ("**/*1", True, 1, ["file1"]), + ( + "**/*1", + True, + 2, + ["file1", "subdir0/subfile1", "subdir1/subfile1", "subdir1/subfile2"], + ), + ("**/*1", False, 2, ["file1", "subdir0/subfile1", "subdir1/subfile1"]), + ("**/subdir0", False, None, []), + ("**/subdir0", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]), + ("**/subdir0/nested*", False, 2, []), + ("**/subdir0/nested*", True, 2, ["nestedfile"]), + ("subdir[1-2]", False, None, []), + ("subdir[1-2]", True, None, ["subfile1", "subfile2", "nesteddir/nestedfile"]), + ("subdir[1-2]", True, 2, ["subfile1", "subfile2"]), + ("subdir[0-1]", False, None, []), + ( + "subdir[0-1]", + True, + None, + [ + "subdir0/subfile1", + "subdir0/subfile2", + "subdir0/nesteddir/nestedfile", + "subdir1/subfile1", + "subdir1/subfile2", + "subdir1/nesteddir/nestedfile", + ], + ), + ( + "subdir[0-1]/*fil[e]*", + False, + None, + [ + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ( + "subdir[0-1]/*fil[e]*", + True, + None, + [ + "subdir0/subfile1", + "subdir0/subfile2", + "subdir1/subfile1", + "subdir1/subfile2", + ], + ), + ], +} diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/copy.py b/lib/python3.12/site-packages/fsspec/tests/abstract/copy.py new file mode 100644 index 0000000000000000000000000000000000000000..e39e57e5f7d52bfda8ab5e2398b04cc2303630a0 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/copy.py @@ -0,0 +1,557 @@ +from hashlib import md5 +from itertools import product + +import pytest + +from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS + + +class AbstractCopyTests: + def test_copy_file_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1a + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + target_file2 = fs_join(target, "file2") + target_subfile1 = fs_join(target, "subfile1") + + # Copy from source directory + fs.cp(fs_join(source, "file2"), target) + assert fs.isfile(target_file2) + + # Copy from sub directory + fs.cp(fs_join(source, "subdir", "subfile1"), target) + assert fs.isfile(target_subfile1) + + # Remove copied files + fs.rm([target_file2, target_subfile1]) + assert not fs.exists(target_file2) + assert not fs.exists(target_subfile1) + + # Repeat with trailing slash on target + fs.cp(fs_join(source, "file2"), target + "/") + assert fs.isdir(target) + assert fs.isfile(target_file2) + + fs.cp(fs_join(source, "subdir", "subfile1"), target + "/") + assert fs.isfile(target_subfile1) + + def test_copy_file_to_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 1b + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.cp( + fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir/") + ) # Note trailing slash + assert fs.isdir(target) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_copy_file_to_file_in_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1c + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + fs.cp(fs_join(source, "subdir", "subfile1"), fs_join(target, "newfile")) + assert fs.isfile(fs_join(target, "newfile")) + + def test_copy_file_to_file_in_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 1d + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.cp( + fs_join(source, "subdir", "subfile1"), fs_join(target, "newdir", "newfile") + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "newfile")) + + def test_copy_directory_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1e + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = target + "/" if target_slash else target + + # Without recursive does nothing + fs.cp(s, t) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + fs.cp(s, t, recursive=True) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert fs.isdir(fs_join(target, "subdir", "nesteddir")) + assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.cp(s, t, recursive=True, maxdepth=1) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert not fs.exists(fs_join(target, "subdir", "nesteddir")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_copy_directory_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1f + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive does nothing + fs.cp(s, t) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + with pytest.raises(FileNotFoundError): + fs.ls(target) + + # With recursive + fs.cp(s, t, recursive=True) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.cp(s, t, recursive=True, maxdepth=1) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + def test_copy_glob_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 1g + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + # Without recursive + fs.cp(fs_join(source, "subdir", "*"), t) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.isdir(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.cp( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_copy_glob_to_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 1h + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for target_slash in [False, True]: + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive + fs.cp(fs_join(source, "subdir", "*"), t) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.cp(fs_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.cp( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + @pytest.mark.parametrize( + GLOB_EDGE_CASES_TESTS["argnames"], + GLOB_EDGE_CASES_TESTS["argvalues"], + ) + def test_copy_glob_edge_cases( + self, + path, + recursive, + maxdepth, + expected, + fs, + fs_join, + fs_glob_edge_cases_files, + fs_target, + fs_sanitize_path, + ): + # Copy scenario 1g + source = fs_glob_edge_cases_files + + target = fs_target + + for new_dir, target_slash in product([True, False], [True, False]): + fs.mkdir(target) + + t = fs_join(target, "newdir") if new_dir else target + t = t + "/" if target_slash else t + + fs.copy(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth) + + output = fs.find(target) + if new_dir: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected + ] + else: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, p)) for p in expected + ] + assert sorted(output) == sorted(prefixed_expected) + + try: + fs.rm(target, recursive=True) + except FileNotFoundError: + pass + + def test_copy_list_of_files_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + fs_target, + supports_empty_directories, + ): + # Copy scenario 2a + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + fs.cp(source_files, t) + assert fs.isfile(fs_join(target, "file1")) + assert fs.isfile(fs_join(target, "file2")) + assert fs.isfile(fs_join(target, "subfile1")) + + fs.rm( + [ + fs_join(target, "file1"), + fs_join(target, "file2"), + fs_join(target, "subfile1"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_copy_list_of_files_to_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # Copy scenario 2b + source = fs_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + fs.cp(source_files, fs_join(target, "newdir") + "/") # Note trailing slash + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "file1")) + assert fs.isfile(fs_join(target, "newdir", "file2")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_copy_two_files_new_directory( + self, fs, fs_join, fs_bulk_operations_scenario_0, fs_target + ): + # This is a duplicate of test_copy_list_of_files_to_new_directory and + # can eventually be removed. + source = fs_bulk_operations_scenario_0 + + target = fs_target + assert not fs.exists(target) + fs.cp([fs_join(source, "file1"), fs_join(source, "file2")], target) + + assert fs.isdir(target) + assert fs.isfile(fs_join(target, "file1")) + assert fs.isfile(fs_join(target, "file2")) + + def test_copy_directory_without_files_with_same_name_prefix( + self, + fs, + fs_join, + fs_target, + fs_dir_and_file_with_same_name_prefix, + supports_empty_directories, + ): + # Create the test dirs + source = fs_dir_and_file_with_same_name_prefix + target = fs_target + + # Test without glob + fs.cp(fs_join(source, "subdir"), target, recursive=True) + + assert fs.isfile(fs_join(target, "subfile.txt")) + assert not fs.isfile(fs_join(target, "subdir.txt")) + + fs.rm([fs_join(target, "subfile.txt")]) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + assert not fs.exists(target) + + # Test with glob + fs.cp(fs_join(source, "subdir*"), target, recursive=True) + + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile.txt")) + assert fs.isfile(fs_join(target, "subdir.txt")) + + def test_copy_with_source_and_destination_as_list( + self, fs, fs_target, fs_join, fs_10_files_with_hashed_names + ): + # Create the test dir + source = fs_10_files_with_hashed_names + target = fs_target + + # Create list of files for source and destination + source_files = [] + destination_files = [] + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + source_files.append(fs_join(source, f"{hashed_i}.txt")) + destination_files.append(fs_join(target, f"{hashed_i}.txt")) + + # Copy and assert order was kept + fs.copy(path1=source_files, path2=destination_files) + + for i in range(10): + file_content = fs.cat(destination_files[i]).decode("utf-8") + assert file_content == str(i) diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/get.py b/lib/python3.12/site-packages/fsspec/tests/abstract/get.py new file mode 100644 index 0000000000000000000000000000000000000000..851ab81ee581e74cac41c64c83ef0af75826d6b0 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/get.py @@ -0,0 +1,587 @@ +from hashlib import md5 +from itertools import product + +import pytest + +from fsspec.implementations.local import make_path_posix +from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS + + +class AbstractGetTests: + def test_get_file_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1a + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + assert local_fs.isdir(target) + + target_file2 = local_join(target, "file2") + target_subfile1 = local_join(target, "subfile1") + + # Copy from source directory + fs.get(fs_join(source, "file2"), target) + assert local_fs.isfile(target_file2) + + # Copy from sub directory + fs.get(fs_join(source, "subdir", "subfile1"), target) + assert local_fs.isfile(target_subfile1) + + # Remove copied files + local_fs.rm([target_file2, target_subfile1]) + assert not local_fs.exists(target_file2) + assert not local_fs.exists(target_subfile1) + + # Repeat with trailing slash on target + fs.get(fs_join(source, "file2"), target + "/") + assert local_fs.isdir(target) + assert local_fs.isfile(target_file2) + + fs.get(fs_join(source, "subdir", "subfile1"), target + "/") + assert local_fs.isfile(target_subfile1) + + def test_get_file_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1b + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + fs.get( + fs_join(source, "subdir", "subfile1"), local_join(target, "newdir/") + ) # Note trailing slash + + assert local_fs.isdir(target) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + + def test_get_file_to_file_in_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1c + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + fs.get(fs_join(source, "subdir", "subfile1"), local_join(target, "newfile")) + assert local_fs.isfile(local_join(target, "newfile")) + + def test_get_file_to_file_in_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1d + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + fs.get( + fs_join(source, "subdir", "subfile1"), + local_join(target, "newdir", "newfile"), + ) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "newfile")) + + def test_get_directory_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1e + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + assert local_fs.isdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = target + "/" if target_slash else target + + # Without recursive does nothing + fs.get(s, t) + assert local_fs.ls(target) == [] + + # With recursive + fs.get(s, t, recursive=True) + if source_slash: + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert local_fs.isdir(local_join(target, "nesteddir")) + assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + local_join(target, "nesteddir"), + ], + recursive=True, + ) + else: + assert local_fs.isdir(local_join(target, "subdir")) + assert local_fs.isfile(local_join(target, "subdir", "subfile1")) + assert local_fs.isfile(local_join(target, "subdir", "subfile2")) + assert local_fs.isdir(local_join(target, "subdir", "nesteddir")) + assert local_fs.isfile( + local_join(target, "subdir", "nesteddir", "nestedfile") + ) + + local_fs.rm(local_join(target, "subdir"), recursive=True) + assert local_fs.ls(target) == [] + + # Limit recursive by maxdepth + fs.get(s, t, recursive=True, maxdepth=1) + if source_slash: + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert not local_fs.exists(local_join(target, "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + ], + recursive=True, + ) + else: + assert local_fs.isdir(local_join(target, "subdir")) + assert local_fs.isfile(local_join(target, "subdir", "subfile1")) + assert local_fs.isfile(local_join(target, "subdir", "subfile2")) + assert not local_fs.exists(local_join(target, "subdir", "nesteddir")) + + local_fs.rm(local_join(target, "subdir"), recursive=True) + assert local_fs.ls(target) == [] + + def test_get_directory_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1f + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = local_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive does nothing + fs.get(s, t) + assert local_fs.ls(target) == [] + + # With recursive + fs.get(s, t, recursive=True) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert local_fs.isdir(local_join(target, "newdir", "nesteddir")) + assert local_fs.isfile( + local_join(target, "newdir", "nesteddir", "nestedfile") + ) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert local_fs.ls(target) == [] + + # Limit recursive by maxdepth + fs.get(s, t, recursive=True, maxdepth=1) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert not local_fs.exists(local_join(target, "newdir")) + + def test_get_glob_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1g + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + # Without recursive + fs.get(fs_join(source, "subdir", "*"), t) + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert not local_fs.isdir(local_join(target, "nesteddir")) + assert not local_fs.exists(local_join(target, "nesteddir", "nestedfile")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.get(fs_join(source, "subdir", glob), t, recursive=recursive) + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert local_fs.isdir(local_join(target, "nesteddir")) + assert local_fs.isfile(local_join(target, "nesteddir", "nestedfile")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + local_join(target, "nesteddir"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + # Limit recursive by maxdepth + fs.get( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert local_fs.isfile(local_join(target, "subfile1")) + assert local_fs.isfile(local_join(target, "subfile2")) + assert not local_fs.exists(local_join(target, "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + + local_fs.rm( + [ + local_join(target, "subfile1"), + local_join(target, "subfile2"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + def test_get_glob_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1h + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + for target_slash in [False, True]: + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive + fs.get(fs_join(source, "subdir", "*"), t) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) + assert not local_fs.exists( + local_join(target, "newdir", "nesteddir", "nestedfile") + ) + assert not local_fs.exists(local_join(target, "subdir")) + assert not local_fs.exists(local_join(target, "newdir", "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert local_fs.ls(target) == [] + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.get(fs_join(source, "subdir", glob), t, recursive=recursive) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert local_fs.isdir(local_join(target, "newdir", "nesteddir")) + assert local_fs.isfile( + local_join(target, "newdir", "nesteddir", "nestedfile") + ) + assert not local_fs.exists(local_join(target, "subdir")) + assert not local_fs.exists(local_join(target, "newdir", "subdir")) + + local_fs.rm(local_join(target, "newdir"), recursive=True) + assert not local_fs.exists(local_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.get( + fs_join(source, "subdir", glob), t, recursive=recursive, maxdepth=1 + ) + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + assert local_fs.isfile(local_join(target, "newdir", "subfile2")) + assert not local_fs.exists(local_join(target, "newdir", "nesteddir")) + assert not local_fs.exists(local_join(target, "subdir")) + assert not local_fs.exists(local_join(target, "newdir", "subdir")) + + local_fs.rm(local_fs.ls(target, detail=False), recursive=True) + assert not local_fs.exists(local_join(target, "newdir")) + + @pytest.mark.parametrize( + GLOB_EDGE_CASES_TESTS["argnames"], + GLOB_EDGE_CASES_TESTS["argvalues"], + ) + def test_get_glob_edge_cases( + self, + path, + recursive, + maxdepth, + expected, + fs, + fs_join, + fs_glob_edge_cases_files, + local_fs, + local_join, + local_target, + ): + # Copy scenario 1g + source = fs_glob_edge_cases_files + + target = local_target + + for new_dir, target_slash in product([True, False], [True, False]): + local_fs.mkdir(target) + + t = local_join(target, "newdir") if new_dir else target + t = t + "/" if target_slash else t + + fs.get(fs_join(source, path), t, recursive=recursive, maxdepth=maxdepth) + + output = local_fs.find(target) + if new_dir: + prefixed_expected = [ + make_path_posix(local_join(target, "newdir", p)) for p in expected + ] + else: + prefixed_expected = [ + make_path_posix(local_join(target, p)) for p in expected + ] + assert sorted(output) == sorted(prefixed_expected) + + try: + local_fs.rm(target, recursive=True) + except FileNotFoundError: + pass + + def test_get_list_of_files_to_existing_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 2a + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + fs.get(source_files, t) + assert local_fs.isfile(local_join(target, "file1")) + assert local_fs.isfile(local_join(target, "file2")) + assert local_fs.isfile(local_join(target, "subfile1")) + + local_fs.rm( + [ + local_join(target, "file1"), + local_join(target, "file2"), + local_join(target, "subfile1"), + ], + recursive=True, + ) + assert local_fs.ls(target) == [] + + def test_get_list_of_files_to_new_directory( + self, + fs, + fs_join, + fs_bulk_operations_scenario_0, + local_fs, + local_join, + local_target, + ): + # Copy scenario 2b + source = fs_bulk_operations_scenario_0 + + target = local_target + local_fs.mkdir(target) + + source_files = [ + fs_join(source, "file1"), + fs_join(source, "file2"), + fs_join(source, "subdir", "subfile1"), + ] + + fs.get(source_files, local_join(target, "newdir") + "/") # Note trailing slash + assert local_fs.isdir(local_join(target, "newdir")) + assert local_fs.isfile(local_join(target, "newdir", "file1")) + assert local_fs.isfile(local_join(target, "newdir", "file2")) + assert local_fs.isfile(local_join(target, "newdir", "subfile1")) + + def test_get_directory_recursive( + self, fs, fs_join, fs_path, local_fs, local_join, local_target + ): + # https://github.com/fsspec/filesystem_spec/issues/1062 + # Recursive cp/get/put of source directory into non-existent target directory. + src = fs_join(fs_path, "src") + src_file = fs_join(src, "file") + fs.mkdir(src) + fs.touch(src_file) + + target = local_target + + # get without slash + assert not local_fs.exists(target) + for loop in range(2): + fs.get(src, target, recursive=True) + assert local_fs.isdir(target) + + if loop == 0: + assert local_fs.isfile(local_join(target, "file")) + assert not local_fs.exists(local_join(target, "src")) + else: + assert local_fs.isfile(local_join(target, "file")) + assert local_fs.isdir(local_join(target, "src")) + assert local_fs.isfile(local_join(target, "src", "file")) + + local_fs.rm(target, recursive=True) + + # get with slash + assert not local_fs.exists(target) + for loop in range(2): + fs.get(src + "/", target, recursive=True) + assert local_fs.isdir(target) + assert local_fs.isfile(local_join(target, "file")) + assert not local_fs.exists(local_join(target, "src")) + + def test_get_directory_without_files_with_same_name_prefix( + self, + fs, + fs_join, + local_fs, + local_join, + local_target, + fs_dir_and_file_with_same_name_prefix, + ): + # Create the test dirs + source = fs_dir_and_file_with_same_name_prefix + target = local_target + + # Test without glob + fs.get(fs_join(source, "subdir"), target, recursive=True) + + assert local_fs.isfile(local_join(target, "subfile.txt")) + assert not local_fs.isfile(local_join(target, "subdir.txt")) + + local_fs.rm([local_join(target, "subfile.txt")]) + assert local_fs.ls(target) == [] + + # Test with glob + fs.get(fs_join(source, "subdir*"), target, recursive=True) + + assert local_fs.isdir(local_join(target, "subdir")) + assert local_fs.isfile(local_join(target, "subdir", "subfile.txt")) + assert local_fs.isfile(local_join(target, "subdir.txt")) + + def test_get_with_source_and_destination_as_list( + self, + fs, + fs_join, + local_fs, + local_join, + local_target, + fs_10_files_with_hashed_names, + ): + # Create the test dir + source = fs_10_files_with_hashed_names + target = local_target + + # Create list of files for source and destination + source_files = [] + destination_files = [] + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + source_files.append(fs_join(source, f"{hashed_i}.txt")) + destination_files.append( + make_path_posix(local_join(target, f"{hashed_i}.txt")) + ) + + # Copy and assert order was kept + fs.get(rpath=source_files, lpath=destination_files) + + for i in range(10): + file_content = local_fs.cat(destination_files[i]).decode("utf-8") + assert file_content == str(i) diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/mv.py b/lib/python3.12/site-packages/fsspec/tests/abstract/mv.py new file mode 100644 index 0000000000000000000000000000000000000000..39f6caa3de815e024fa84de2acecc986c823ed29 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/mv.py @@ -0,0 +1,57 @@ +import os + +import pytest + +import fsspec + + +def test_move_raises_error_with_tmpdir(tmpdir): + # Create a file in the temporary directory + source = tmpdir.join("source_file.txt") + source.write("content") + + # Define a destination that simulates a protected or invalid path + destination = tmpdir.join("non_existent_directory/destination_file.txt") + + # Instantiate the filesystem (assuming the local file system interface) + fs = fsspec.filesystem("file") + + # Use the actual file paths as string + with pytest.raises(FileNotFoundError): + fs.mv(str(source), str(destination)) + + +@pytest.mark.parametrize("recursive", (True, False)) +def test_move_raises_error_with_tmpdir_permission(recursive, tmpdir): + # Create a file in the temporary directory + source = tmpdir.join("source_file.txt") + source.write("content") + + # Create a protected directory (non-writable) + protected_dir = tmpdir.mkdir("protected_directory") + protected_path = str(protected_dir) + + # Set the directory to read-only + if os.name == "nt": + os.system(f'icacls "{protected_path}" /deny Everyone:(W)') + else: + os.chmod(protected_path, 0o555) # Sets the directory to read-only + + # Define a destination inside the protected directory + destination = protected_dir.join("destination_file.txt") + + # Instantiate the filesystem (assuming the local file system interface) + fs = fsspec.filesystem("file") + + # Try to move the file to the read-only directory, expecting a permission error + with pytest.raises(PermissionError): + fs.mv(str(source), str(destination), recursive=recursive) + + # Assert the file was not created in the destination + assert not os.path.exists(destination) + + # Cleanup: Restore permissions so the directory can be cleaned up + if os.name == "nt": + os.system(f'icacls "{protected_path}" /remove:d Everyone') + else: + os.chmod(protected_path, 0o755) # Restore write permission for cleanup diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/open.py b/lib/python3.12/site-packages/fsspec/tests/abstract/open.py new file mode 100644 index 0000000000000000000000000000000000000000..bb75ea852276fb8d834345883813b8e27a0ae24c --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/open.py @@ -0,0 +1,11 @@ +import pytest + + +class AbstractOpenTests: + def test_open_exclusive(self, fs, fs_target): + with fs.open(fs_target, "wb") as f: + f.write(b"data") + with fs.open(fs_target, "rb") as f: + assert f.read() == b"data" + with pytest.raises(FileExistsError): + fs.open(fs_target, "xb") diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/pipe.py b/lib/python3.12/site-packages/fsspec/tests/abstract/pipe.py new file mode 100644 index 0000000000000000000000000000000000000000..8ecca96e9d23ff268a253c48269d5cca451ea270 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/pipe.py @@ -0,0 +1,11 @@ +import pytest + + +class AbstractPipeTests: + def test_pipe_exclusive(self, fs, fs_target): + fs.pipe_file(fs_target, b"data") + assert fs.cat_file(fs_target) == b"data" + with pytest.raises(FileExistsError): + fs.pipe_file(fs_target, b"data", mode="create") + fs.pipe_file(fs_target, b"new data", mode="overwrite") + assert fs.cat_file(fs_target) == b"new data" diff --git a/lib/python3.12/site-packages/fsspec/tests/abstract/put.py b/lib/python3.12/site-packages/fsspec/tests/abstract/put.py new file mode 100644 index 0000000000000000000000000000000000000000..9fc349977f0384d9fc86126498be5c6ad99a21d3 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/tests/abstract/put.py @@ -0,0 +1,591 @@ +from hashlib import md5 +from itertools import product + +import pytest + +from fsspec.tests.abstract.common import GLOB_EDGE_CASES_TESTS + + +class AbstractPutTests: + def test_put_file_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 1a + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + target_file2 = fs_join(target, "file2") + target_subfile1 = fs_join(target, "subfile1") + + # Copy from source directory + fs.put(local_join(source, "file2"), target) + assert fs.isfile(target_file2) + + # Copy from sub directory + fs.put(local_join(source, "subdir", "subfile1"), target) + assert fs.isfile(target_subfile1) + + # Remove copied files + fs.rm([target_file2, target_subfile1]) + assert not fs.exists(target_file2) + assert not fs.exists(target_subfile1) + + # Repeat with trailing slash on target + fs.put(local_join(source, "file2"), target + "/") + assert fs.isdir(target) + assert fs.isfile(target_file2) + + fs.put(local_join(source, "subdir", "subfile1"), target + "/") + assert fs.isfile(target_subfile1) + + def test_put_file_to_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 1b + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.put( + local_join(source, "subdir", "subfile1"), fs_join(target, "newdir/") + ) # Note trailing slash + assert fs.isdir(target) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_put_file_to_file_in_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + supports_empty_directories, + local_bulk_operations_scenario_0, + ): + # Copy scenario 1c + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + fs.touch(fs_join(target, "dummy")) + assert fs.isdir(target) + + fs.put(local_join(source, "subdir", "subfile1"), fs_join(target, "newfile")) + assert fs.isfile(fs_join(target, "newfile")) + + def test_put_file_to_file_in_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 1d + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + fs.put( + local_join(source, "subdir", "subfile1"), + fs_join(target, "newdir", "newfile"), + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "newfile")) + + def test_put_directory_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 1e + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = target + "/" if target_slash else target + + # Without recursive does nothing + fs.put(s, t) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + fs.put(s, t, recursive=True) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert fs.isdir(fs_join(target, "subdir", "nesteddir")) + assert fs.isfile(fs_join(target, "subdir", "nesteddir", "nestedfile")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.put(s, t, recursive=True, maxdepth=1) + if source_slash: + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + else: + assert fs.isdir(fs_join(target, "subdir")) + assert fs.isfile(fs_join(target, "subdir", "subfile1")) + assert fs.isfile(fs_join(target, "subdir", "subfile2")) + assert not fs.exists(fs_join(target, "subdir", "nesteddir")) + + fs.rm(fs_join(target, "subdir"), recursive=True) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_put_directory_to_new_directory( + self, + fs, + fs_join, + fs_target, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 1f + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for source_slash, target_slash in zip([False, True], [False, True]): + s = fs_join(source, "subdir") + if source_slash: + s += "/" + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive does nothing + fs.put(s, t) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + with pytest.raises(FileNotFoundError): + fs.ls(target) + + # With recursive + fs.put(s, t, recursive=True) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.put(s, t, recursive=True, maxdepth=1) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + def test_put_glob_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + supports_empty_directories, + local_bulk_operations_scenario_0, + ): + # Copy scenario 1g + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + # Without recursive + fs.put(local_join(source, "subdir", "*"), t) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.isdir(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.put(local_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert fs.isdir(fs_join(target, "nesteddir")) + assert fs.isfile(fs_join(target, "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + fs_join(target, "nesteddir"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + # Limit recursive by maxdepth + fs.put( + local_join(source, "subdir", glob), + t, + recursive=recursive, + maxdepth=1, + ) + assert fs.isfile(fs_join(target, "subfile1")) + assert fs.isfile(fs_join(target, "subfile2")) + assert not fs.exists(fs_join(target, "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + + fs.rm( + [ + fs_join(target, "subfile1"), + fs_join(target, "subfile2"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_put_glob_to_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 1h + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + for target_slash in [False, True]: + t = fs_join(target, "newdir") + if target_slash: + t += "/" + + # Without recursive + fs.put(local_join(source, "subdir", "*"), t) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # With recursive + for glob, recursive in zip(["*", "**"], [True, False]): + fs.put(local_join(source, "subdir", glob), t, recursive=recursive) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert fs.isdir(fs_join(target, "newdir", "nesteddir")) + assert fs.isfile(fs_join(target, "newdir", "nesteddir", "nestedfile")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + # Limit recursive by maxdepth + fs.put( + local_join(source, "subdir", glob), + t, + recursive=recursive, + maxdepth=1, + ) + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + assert fs.isfile(fs_join(target, "newdir", "subfile2")) + assert not fs.exists(fs_join(target, "newdir", "nesteddir")) + assert not fs.exists(fs_join(target, "subdir")) + assert not fs.exists(fs_join(target, "newdir", "subdir")) + + fs.rm(fs_join(target, "newdir"), recursive=True) + assert not fs.exists(fs_join(target, "newdir")) + + @pytest.mark.parametrize( + GLOB_EDGE_CASES_TESTS["argnames"], + GLOB_EDGE_CASES_TESTS["argvalues"], + ) + def test_put_glob_edge_cases( + self, + path, + recursive, + maxdepth, + expected, + fs, + fs_join, + fs_target, + local_glob_edge_cases_files, + local_join, + fs_sanitize_path, + ): + # Copy scenario 1g + source = local_glob_edge_cases_files + + target = fs_target + + for new_dir, target_slash in product([True, False], [True, False]): + fs.mkdir(target) + + t = fs_join(target, "newdir") if new_dir else target + t = t + "/" if target_slash else t + + fs.put(local_join(source, path), t, recursive=recursive, maxdepth=maxdepth) + + output = fs.find(target) + if new_dir: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, "newdir", p)) for p in expected + ] + else: + prefixed_expected = [ + fs_sanitize_path(fs_join(target, p)) for p in expected + ] + assert sorted(output) == sorted(prefixed_expected) + + try: + fs.rm(target, recursive=True) + except FileNotFoundError: + pass + + def test_put_list_of_files_to_existing_directory( + self, + fs, + fs_join, + fs_target, + local_join, + local_bulk_operations_scenario_0, + supports_empty_directories, + ): + # Copy scenario 2a + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + if not supports_empty_directories: + # Force target directory to exist by adding a dummy file + dummy = fs_join(target, "dummy") + fs.touch(dummy) + assert fs.isdir(target) + + source_files = [ + local_join(source, "file1"), + local_join(source, "file2"), + local_join(source, "subdir", "subfile1"), + ] + + for target_slash in [False, True]: + t = target + "/" if target_slash else target + + fs.put(source_files, t) + assert fs.isfile(fs_join(target, "file1")) + assert fs.isfile(fs_join(target, "file2")) + assert fs.isfile(fs_join(target, "subfile1")) + + fs.rm( + [ + fs_join(target, "file1"), + fs_join(target, "file2"), + fs_join(target, "subfile1"), + ], + recursive=True, + ) + assert fs.ls(target, detail=False) == ( + [] if supports_empty_directories else [dummy] + ) + + def test_put_list_of_files_to_new_directory( + self, fs, fs_join, fs_target, local_join, local_bulk_operations_scenario_0 + ): + # Copy scenario 2b + source = local_bulk_operations_scenario_0 + + target = fs_target + fs.mkdir(target) + + source_files = [ + local_join(source, "file1"), + local_join(source, "file2"), + local_join(source, "subdir", "subfile1"), + ] + + fs.put(source_files, fs_join(target, "newdir") + "/") # Note trailing slash + assert fs.isdir(fs_join(target, "newdir")) + assert fs.isfile(fs_join(target, "newdir", "file1")) + assert fs.isfile(fs_join(target, "newdir", "file2")) + assert fs.isfile(fs_join(target, "newdir", "subfile1")) + + def test_put_directory_recursive( + self, fs, fs_join, fs_target, local_fs, local_join, local_path + ): + # https://github.com/fsspec/filesystem_spec/issues/1062 + # Recursive cp/get/put of source directory into non-existent target directory. + src = local_join(local_path, "src") + src_file = local_join(src, "file") + local_fs.mkdir(src) + local_fs.touch(src_file) + + target = fs_target + + # put without slash + assert not fs.exists(target) + for loop in range(2): + fs.put(src, target, recursive=True) + assert fs.isdir(target) + + if loop == 0: + assert fs.isfile(fs_join(target, "file")) + assert not fs.exists(fs_join(target, "src")) + else: + assert fs.isfile(fs_join(target, "file")) + assert fs.isdir(fs_join(target, "src")) + assert fs.isfile(fs_join(target, "src", "file")) + + fs.rm(target, recursive=True) + + # put with slash + assert not fs.exists(target) + for loop in range(2): + fs.put(src + "/", target, recursive=True) + assert fs.isdir(target) + assert fs.isfile(fs_join(target, "file")) + assert not fs.exists(fs_join(target, "src")) + + def test_put_directory_without_files_with_same_name_prefix( + self, + fs, + fs_join, + fs_target, + local_join, + local_dir_and_file_with_same_name_prefix, + supports_empty_directories, + ): + # Create the test dirs + source = local_dir_and_file_with_same_name_prefix + target = fs_target + + # Test without glob + fs.put(local_join(source, "subdir"), fs_target, recursive=True) + + assert fs.isfile(fs_join(fs_target, "subfile.txt")) + assert not fs.isfile(fs_join(fs_target, "subdir.txt")) + + fs.rm([fs_join(target, "subfile.txt")]) + if supports_empty_directories: + assert fs.ls(target) == [] + else: + assert not fs.exists(target) + + # Test with glob + fs.put(local_join(source, "subdir*"), fs_target, recursive=True) + + assert fs.isdir(fs_join(fs_target, "subdir")) + assert fs.isfile(fs_join(fs_target, "subdir", "subfile.txt")) + assert fs.isfile(fs_join(fs_target, "subdir.txt")) + + def test_copy_with_source_and_destination_as_list( + self, fs, fs_target, fs_join, local_join, local_10_files_with_hashed_names + ): + # Create the test dir + source = local_10_files_with_hashed_names + target = fs_target + + # Create list of files for source and destination + source_files = [] + destination_files = [] + for i in range(10): + hashed_i = md5(str(i).encode("utf-8")).hexdigest() + source_files.append(local_join(source, f"{hashed_i}.txt")) + destination_files.append(fs_join(target, f"{hashed_i}.txt")) + + # Copy and assert order was kept + fs.put(lpath=source_files, rpath=destination_files) + + for i in range(10): + file_content = fs.cat(destination_files[i]).decode("utf-8") + assert file_content == str(i) diff --git a/lib/python3.12/site-packages/fsspec/transaction.py b/lib/python3.12/site-packages/fsspec/transaction.py new file mode 100644 index 0000000000000000000000000000000000000000..77293f63ecc5f611e19d849ef236d53e9c258efc --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/transaction.py @@ -0,0 +1,90 @@ +from collections import deque + + +class Transaction: + """Filesystem transaction write context + + Gathers files for deferred commit or discard, so that several write + operations can be finalized semi-atomically. This works by having this + instance as the ``.transaction`` attribute of the given filesystem + """ + + def __init__(self, fs, **kwargs): + """ + Parameters + ---------- + fs: FileSystem instance + """ + self.fs = fs + self.files = deque() + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """End transaction and commit, if exit is not due to exception""" + # only commit if there was no exception + self.complete(commit=exc_type is None) + if self.fs: + self.fs._intrans = False + self.fs._transaction = None + self.fs = None + + def start(self): + """Start a transaction on this FileSystem""" + self.files = deque() # clean up after previous failed completions + self.fs._intrans = True + + def complete(self, commit=True): + """Finish transaction: commit or discard all deferred files""" + while self.files: + f = self.files.popleft() + if commit: + f.commit() + else: + f.discard() + self.fs._intrans = False + self.fs._transaction = None + self.fs = None + + +class FileActor: + def __init__(self): + self.files = [] + + def commit(self): + for f in self.files: + f.commit() + self.files.clear() + + def discard(self): + for f in self.files: + f.discard() + self.files.clear() + + def append(self, f): + self.files.append(f) + + +class DaskTransaction(Transaction): + def __init__(self, fs): + """ + Parameters + ---------- + fs: FileSystem instance + """ + import distributed + + super().__init__(fs) + client = distributed.default_client() + self.files = client.submit(FileActor, actor=True).result() + + def complete(self, commit=True): + """Finish transaction: commit or discard all deferred files""" + if commit: + self.files.commit().result() + else: + self.files.discard().result() + self.fs._intrans = False + self.fs = None diff --git a/lib/python3.12/site-packages/fsspec/utils.py b/lib/python3.12/site-packages/fsspec/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..917af8d19c560212a9757d11a16213f54ec77356 --- /dev/null +++ b/lib/python3.12/site-packages/fsspec/utils.py @@ -0,0 +1,739 @@ +from __future__ import annotations + +import contextlib +import logging +import math +import os +import re +import sys +import tempfile +from functools import partial +from hashlib import md5 +from importlib.metadata import version +from typing import ( + IO, + TYPE_CHECKING, + Any, + Callable, + Iterable, + Iterator, + Sequence, + TypeVar, +) +from urllib.parse import urlsplit + +if TYPE_CHECKING: + import pathlib + + from typing_extensions import TypeGuard + + from fsspec.spec import AbstractFileSystem + + +DEFAULT_BLOCK_SIZE = 5 * 2**20 + +T = TypeVar("T") + + +def infer_storage_options( + urlpath: str, inherit_storage_options: dict[str, Any] | None = None +) -> dict[str, Any]: + """Infer storage options from URL path and merge it with existing storage + options. + + Parameters + ---------- + urlpath: str or unicode + Either local absolute file path or URL (hdfs://namenode:8020/file.csv) + inherit_storage_options: dict (optional) + Its contents will get merged with the inferred information from the + given path + + Returns + ------- + Storage options dict. + + Examples + -------- + >>> infer_storage_options('/mnt/datasets/test.csv') # doctest: +SKIP + {"protocol": "file", "path", "/mnt/datasets/test.csv"} + >>> infer_storage_options( + ... 'hdfs://username:pwd@node:123/mnt/datasets/test.csv?q=1', + ... inherit_storage_options={'extra': 'value'}, + ... ) # doctest: +SKIP + {"protocol": "hdfs", "username": "username", "password": "pwd", + "host": "node", "port": 123, "path": "/mnt/datasets/test.csv", + "url_query": "q=1", "extra": "value"} + """ + # Handle Windows paths including disk name in this special case + if ( + re.match(r"^[a-zA-Z]:[\\/]", urlpath) + or re.match(r"^[a-zA-Z0-9]+://", urlpath) is None + ): + return {"protocol": "file", "path": urlpath} + + parsed_path = urlsplit(urlpath) + protocol = parsed_path.scheme or "file" + if parsed_path.fragment: + path = "#".join([parsed_path.path, parsed_path.fragment]) + else: + path = parsed_path.path + if protocol == "file": + # Special case parsing file protocol URL on Windows according to: + # https://msdn.microsoft.com/en-us/library/jj710207.aspx + windows_path = re.match(r"^/([a-zA-Z])[:|]([\\/].*)$", path) + if windows_path: + drive, path = windows_path.groups() + path = f"{drive}:{path}" + + if protocol in ["http", "https"]: + # for HTTP, we don't want to parse, as requests will anyway + return {"protocol": protocol, "path": urlpath} + + options: dict[str, Any] = {"protocol": protocol, "path": path} + + if parsed_path.netloc: + # Parse `hostname` from netloc manually because `parsed_path.hostname` + # lowercases the hostname which is not always desirable (e.g. in S3): + # https://github.com/dask/dask/issues/1417 + options["host"] = parsed_path.netloc.rsplit("@", 1)[-1].rsplit(":", 1)[0] + + if protocol in ("s3", "s3a", "gcs", "gs"): + options["path"] = options["host"] + options["path"] + else: + options["host"] = options["host"] + if parsed_path.port: + options["port"] = parsed_path.port + if parsed_path.username: + options["username"] = parsed_path.username + if parsed_path.password: + options["password"] = parsed_path.password + + if parsed_path.query: + options["url_query"] = parsed_path.query + if parsed_path.fragment: + options["url_fragment"] = parsed_path.fragment + + if inherit_storage_options: + update_storage_options(options, inherit_storage_options) + + return options + + +def update_storage_options( + options: dict[str, Any], inherited: dict[str, Any] | None = None +) -> None: + if not inherited: + inherited = {} + collisions = set(options) & set(inherited) + if collisions: + for collision in collisions: + if options.get(collision) != inherited.get(collision): + raise KeyError( + f"Collision between inferred and specified storage " + f"option:\n{collision}" + ) + options.update(inherited) + + +# Compression extensions registered via fsspec.compression.register_compression +compressions: dict[str, str] = {} + + +def infer_compression(filename: str) -> str | None: + """Infer compression, if available, from filename. + + Infer a named compression type, if registered and available, from filename + extension. This includes builtin (gz, bz2, zip) compressions, as well as + optional compressions. See fsspec.compression.register_compression. + """ + extension = os.path.splitext(filename)[-1].strip(".").lower() + if extension in compressions: + return compressions[extension] + return None + + +def build_name_function(max_int: float) -> Callable[[int], str]: + """Returns a function that receives a single integer + and returns it as a string padded by enough zero characters + to align with maximum possible integer + + >>> name_f = build_name_function(57) + + >>> name_f(7) + '07' + >>> name_f(31) + '31' + >>> build_name_function(1000)(42) + '0042' + >>> build_name_function(999)(42) + '042' + >>> build_name_function(0)(0) + '0' + """ + # handle corner cases max_int is 0 or exact power of 10 + max_int += 1e-8 + + pad_length = int(math.ceil(math.log10(max_int))) + + def name_function(i: int) -> str: + return str(i).zfill(pad_length) + + return name_function + + +def seek_delimiter(file: IO[bytes], delimiter: bytes, blocksize: int) -> bool: + r"""Seek current file to file start, file end, or byte after delimiter seq. + + Seeks file to next chunk delimiter, where chunks are defined on file start, + a delimiting sequence, and file end. Use file.tell() to see location afterwards. + Note that file start is a valid split, so must be at offset > 0 to seek for + delimiter. + + Parameters + ---------- + file: a file + delimiter: bytes + a delimiter like ``b'\n'`` or message sentinel, matching file .read() type + blocksize: int + Number of bytes to read from the file at once. + + + Returns + ------- + Returns True if a delimiter was found, False if at file start or end. + + """ + + if file.tell() == 0: + # beginning-of-file, return without seek + return False + + # Interface is for binary IO, with delimiter as bytes, but initialize last + # with result of file.read to preserve compatibility with text IO. + last: bytes | None = None + while True: + current = file.read(blocksize) + if not current: + # end-of-file without delimiter + return False + full = last + current if last else current + try: + if delimiter in full: + i = full.index(delimiter) + file.seek(file.tell() - (len(full) - i) + len(delimiter)) + return True + elif len(current) < blocksize: + # end-of-file without delimiter + return False + except (OSError, ValueError): + pass + last = full[-len(delimiter) :] + + +def read_block( + f: IO[bytes], + offset: int, + length: int | None, + delimiter: bytes | None = None, + split_before: bool = False, +) -> bytes: + """Read a block of bytes from a file + + Parameters + ---------- + f: File + Open file + offset: int + Byte offset to start read + length: int + Number of bytes to read, read through end of file if None + delimiter: bytes (optional) + Ensure reading starts and stops at delimiter bytestring + split_before: bool (optional) + Start/stop read *before* delimiter bytestring. + + + If using the ``delimiter=`` keyword argument we ensure that the read + starts and stops at delimiter boundaries that follow the locations + ``offset`` and ``offset + length``. If ``offset`` is zero then we + start at zero, regardless of delimiter. The bytestring returned WILL + include the terminating delimiter string. + + Examples + -------- + + >>> from io import BytesIO # doctest: +SKIP + >>> f = BytesIO(b'Alice, 100\\nBob, 200\\nCharlie, 300') # doctest: +SKIP + >>> read_block(f, 0, 13) # doctest: +SKIP + b'Alice, 100\\nBo' + + >>> read_block(f, 0, 13, delimiter=b'\\n') # doctest: +SKIP + b'Alice, 100\\nBob, 200\\n' + + >>> read_block(f, 10, 10, delimiter=b'\\n') # doctest: +SKIP + b'Bob, 200\\nCharlie, 300' + """ + if delimiter: + f.seek(offset) + found_start_delim = seek_delimiter(f, delimiter, 2**16) + if length is None: + return f.read() + start = f.tell() + length -= start - offset + + f.seek(start + length) + found_end_delim = seek_delimiter(f, delimiter, 2**16) + end = f.tell() + + # Adjust split location to before delimiter if seek found the + # delimiter sequence, not start or end of file. + if found_start_delim and split_before: + start -= len(delimiter) + + if found_end_delim and split_before: + end -= len(delimiter) + + offset = start + length = end - start + + f.seek(offset) + + # TODO: allow length to be None and read to the end of the file? + assert length is not None + b = f.read(length) + return b + + +def tokenize(*args: Any, **kwargs: Any) -> str: + """Deterministic token + + (modified from dask.base) + + >>> tokenize([1, 2, '3']) + '9d71491b50023b06fc76928e6eddb952' + + >>> tokenize('Hello') == tokenize('Hello') + True + """ + if kwargs: + args += (kwargs,) + try: + h = md5(str(args).encode()) + except ValueError: + # FIPS systems: https://github.com/fsspec/filesystem_spec/issues/380 + h = md5(str(args).encode(), usedforsecurity=False) + return h.hexdigest() + + +def stringify_path(filepath: str | os.PathLike[str] | pathlib.Path) -> str: + """Attempt to convert a path-like object to a string. + + Parameters + ---------- + filepath: object to be converted + + Returns + ------- + filepath_str: maybe a string version of the object + + Notes + ----- + Objects supporting the fspath protocol are coerced according to its + __fspath__ method. + + For backwards compatibility with older Python version, pathlib.Path + objects are specially coerced. + + Any other object is passed through unchanged, which includes bytes, + strings, buffers, or anything else that's not even path-like. + """ + if isinstance(filepath, str): + return filepath + elif hasattr(filepath, "__fspath__"): + return filepath.__fspath__() + elif hasattr(filepath, "path"): + return filepath.path + else: + return filepath # type: ignore[return-value] + + +def make_instance( + cls: Callable[..., T], args: Sequence[Any], kwargs: dict[str, Any] +) -> T: + inst = cls(*args, **kwargs) + inst._determine_worker() # type: ignore[attr-defined] + return inst + + +def common_prefix(paths: Iterable[str]) -> str: + """For a list of paths, find the shortest prefix common to all""" + parts = [p.split("/") for p in paths] + lmax = min(len(p) for p in parts) + end = 0 + for i in range(lmax): + end = all(p[i] == parts[0][i] for p in parts) + if not end: + break + i += end + return "/".join(parts[0][:i]) + + +def other_paths( + paths: list[str], + path2: str | list[str], + exists: bool = False, + flatten: bool = False, +) -> list[str]: + """In bulk file operations, construct a new file tree from a list of files + + Parameters + ---------- + paths: list of str + The input file tree + path2: str or list of str + Root to construct the new list in. If this is already a list of str, we just + assert it has the right number of elements. + exists: bool (optional) + For a str destination, it is already exists (and is a dir), files should + end up inside. + flatten: bool (optional) + Whether to flatten the input directory tree structure so that the output files + are in the same directory. + + Returns + ------- + list of str + """ + + if isinstance(path2, str): + path2 = path2.rstrip("/") + + if flatten: + path2 = ["/".join((path2, p.split("/")[-1])) for p in paths] + else: + cp = common_prefix(paths) + if exists: + cp = cp.rsplit("/", 1)[0] + if not cp and all(not s.startswith("/") for s in paths): + path2 = ["/".join([path2, p]) for p in paths] + else: + path2 = [p.replace(cp, path2, 1) for p in paths] + else: + assert len(paths) == len(path2) + return path2 + + +def is_exception(obj: Any) -> bool: + return isinstance(obj, BaseException) + + +def isfilelike(f: Any) -> TypeGuard[IO[bytes]]: + return all(hasattr(f, attr) for attr in ["read", "close", "tell"]) + + +def get_protocol(url: str) -> str: + url = stringify_path(url) + parts = re.split(r"(\:\:|\://)", url, maxsplit=1) + if len(parts) > 1: + return parts[0] + return "file" + + +def can_be_local(path: str) -> bool: + """Can the given URL be used with open_local?""" + from fsspec import get_filesystem_class + + try: + return getattr(get_filesystem_class(get_protocol(path)), "local_file", False) + except (ValueError, ImportError): + # not in registry or import failed + return False + + +def get_package_version_without_import(name: str) -> str | None: + """For given package name, try to find the version without importing it + + Import and package.__version__ is still the backup here, so an import + *might* happen. + + Returns either the version string, or None if the package + or the version was not readily found. + """ + if name in sys.modules: + mod = sys.modules[name] + if hasattr(mod, "__version__"): + return mod.__version__ + try: + return version(name) + except: # noqa: E722 + pass + try: + import importlib + + mod = importlib.import_module(name) + return mod.__version__ + except (ImportError, AttributeError): + return None + + +def setup_logging( + logger: logging.Logger | None = None, + logger_name: str | None = None, + level: str = "DEBUG", + clear: bool = True, +) -> logging.Logger: + if logger is None and logger_name is None: + raise ValueError("Provide either logger object or logger name") + logger = logger or logging.getLogger(logger_name) + handle = logging.StreamHandler() + formatter = logging.Formatter( + "%(asctime)s - %(name)s - %(levelname)s - %(funcName)s -- %(message)s" + ) + handle.setFormatter(formatter) + if clear: + logger.handlers.clear() + logger.addHandler(handle) + logger.setLevel(level) + return logger + + +def _unstrip_protocol(name: str, fs: AbstractFileSystem) -> str: + return fs.unstrip_protocol(name) + + +def mirror_from( + origin_name: str, methods: Iterable[str] +) -> Callable[[type[T]], type[T]]: + """Mirror attributes and methods from the given + origin_name attribute of the instance to the + decorated class""" + + def origin_getter(method: str, self: Any) -> Any: + origin = getattr(self, origin_name) + return getattr(origin, method) + + def wrapper(cls: type[T]) -> type[T]: + for method in methods: + wrapped_method = partial(origin_getter, method) + setattr(cls, method, property(wrapped_method)) + return cls + + return wrapper + + +@contextlib.contextmanager +def nullcontext(obj: T) -> Iterator[T]: + yield obj + + +def merge_offset_ranges( + paths: list[str], + starts: list[int] | int, + ends: list[int] | int, + max_gap: int = 0, + max_block: int | None = None, + sort: bool = True, +) -> tuple[list[str], list[int], list[int]]: + """Merge adjacent byte-offset ranges when the inter-range + gap is <= `max_gap`, and when the merged byte range does not + exceed `max_block` (if specified). By default, this function + will re-order the input paths and byte ranges to ensure sorted + order. If the user can guarantee that the inputs are already + sorted, passing `sort=False` will skip the re-ordering. + """ + # Check input + if not isinstance(paths, list): + raise TypeError + if not isinstance(starts, list): + starts = [starts] * len(paths) + if not isinstance(ends, list): + ends = [ends] * len(paths) + if len(starts) != len(paths) or len(ends) != len(paths): + raise ValueError + + # Early Return + if len(starts) <= 1: + return paths, starts, ends + + starts = [s or 0 for s in starts] + # Sort by paths and then ranges if `sort=True` + if sort: + paths, starts, ends = ( + list(v) + for v in zip( + *sorted( + zip(paths, starts, ends), + ) + ) + ) + + if paths: + # Loop through the coupled `paths`, `starts`, and + # `ends`, and merge adjacent blocks when appropriate + new_paths = paths[:1] + new_starts = starts[:1] + new_ends = ends[:1] + for i in range(1, len(paths)): + if paths[i] == paths[i - 1] and new_ends[-1] is None: + continue + elif ( + paths[i] != paths[i - 1] + or ((starts[i] - new_ends[-1]) > max_gap) + or (max_block is not None and (ends[i] - new_starts[-1]) > max_block) + ): + # Cannot merge with previous block. + # Add new `paths`, `starts`, and `ends` elements + new_paths.append(paths[i]) + new_starts.append(starts[i]) + new_ends.append(ends[i]) + else: + # Merge with previous block by updating the + # last element of `ends` + new_ends[-1] = ends[i] + return new_paths, new_starts, new_ends + + # `paths` is empty. Just return input lists + return paths, starts, ends + + +def file_size(filelike: IO[bytes]) -> int: + """Find length of any open read-mode file-like""" + pos = filelike.tell() + try: + return filelike.seek(0, 2) + finally: + filelike.seek(pos) + + +@contextlib.contextmanager +def atomic_write(path: str, mode: str = "wb"): + """ + A context manager that opens a temporary file next to `path` and, on exit, + replaces `path` with the temporary file, thereby updating `path` + atomically. + """ + fd, fn = tempfile.mkstemp( + dir=os.path.dirname(path), prefix=os.path.basename(path) + "-" + ) + try: + with open(fd, mode) as fp: + yield fp + except BaseException: + with contextlib.suppress(FileNotFoundError): + os.unlink(fn) + raise + else: + os.replace(fn, path) + + +def _translate(pat, STAR, QUESTION_MARK): + # Copied from: https://github.com/python/cpython/pull/106703. + res: list[str] = [] + add = res.append + i, n = 0, len(pat) + while i < n: + c = pat[i] + i = i + 1 + if c == "*": + # compress consecutive `*` into one + if (not res) or res[-1] is not STAR: + add(STAR) + elif c == "?": + add(QUESTION_MARK) + elif c == "[": + j = i + if j < n and pat[j] == "!": + j = j + 1 + if j < n and pat[j] == "]": + j = j + 1 + while j < n and pat[j] != "]": + j = j + 1 + if j >= n: + add("\\[") + else: + stuff = pat[i:j] + if "-" not in stuff: + stuff = stuff.replace("\\", r"\\") + else: + chunks = [] + k = i + 2 if pat[i] == "!" else i + 1 + while True: + k = pat.find("-", k, j) + if k < 0: + break + chunks.append(pat[i:k]) + i = k + 1 + k = k + 3 + chunk = pat[i:j] + if chunk: + chunks.append(chunk) + else: + chunks[-1] += "-" + # Remove empty ranges -- invalid in RE. + for k in range(len(chunks) - 1, 0, -1): + if chunks[k - 1][-1] > chunks[k][0]: + chunks[k - 1] = chunks[k - 1][:-1] + chunks[k][1:] + del chunks[k] + # Escape backslashes and hyphens for set difference (--). + # Hyphens that create ranges shouldn't be escaped. + stuff = "-".join( + s.replace("\\", r"\\").replace("-", r"\-") for s in chunks + ) + # Escape set operations (&&, ~~ and ||). + stuff = re.sub(r"([&~|])", r"\\\1", stuff) + i = j + 1 + if not stuff: + # Empty range: never match. + add("(?!)") + elif stuff == "!": + # Negated empty range: match any character. + add(".") + else: + if stuff[0] == "!": + stuff = "^" + stuff[1:] + elif stuff[0] in ("^", "["): + stuff = "\\" + stuff + add(f"[{stuff}]") + else: + add(re.escape(c)) + assert i == n + return res + + +def glob_translate(pat): + # Copied from: https://github.com/python/cpython/pull/106703. + # The keyword parameters' values are fixed to: + # recursive=True, include_hidden=True, seps=None + """Translate a pathname with shell wildcards to a regular expression.""" + if os.path.altsep: + seps = os.path.sep + os.path.altsep + else: + seps = os.path.sep + escaped_seps = "".join(map(re.escape, seps)) + any_sep = f"[{escaped_seps}]" if len(seps) > 1 else escaped_seps + not_sep = f"[^{escaped_seps}]" + one_last_segment = f"{not_sep}+" + one_segment = f"{one_last_segment}{any_sep}" + any_segments = f"(?:.+{any_sep})?" + any_last_segments = ".*" + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == "*": + results.append(one_segment if idx < last_part_idx else one_last_segment) + continue + if part == "**": + results.append(any_segments if idx < last_part_idx else any_last_segments) + continue + elif "**" in part: + raise ValueError( + "Invalid pattern: '**' can only be an entire path component" + ) + if part: + results.extend(_translate(part, f"{not_sep}*", not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = "".join(results) + return rf"(?s:{res})\Z" diff --git a/lib/python3.12/site-packages/google/protobuf/__init__.py b/lib/python3.12/site-packages/google/protobuf/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9e9d209b1832667d659a2ad34f336086ca6ed284 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/__init__.py @@ -0,0 +1,10 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# Copyright 2007 Google Inc. All Rights Reserved. + +__version__ = '6.33.5' diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2644f57f5278256fe52a0639d852d4e1ba630aab Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/any.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/any.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8ef40a1597bb504a4399146e2d39abb8847e87f9 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/any.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/any_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/any_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3b47631aacca74c1e63d3c91e9d3defe52fe6ab0 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/any_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/api_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/api_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..52d4dc9e8dea21d66d760bf5dd02a8f5a09b59db Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/api_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a9b6fb2e8172f756f638e9ff6d0ec79eb2bee271 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor_database.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor_database.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4796371f341e98bea69d3405e54273c0999e10c6 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor_database.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor_pool.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor_pool.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..29efb083ad41b148e9f4a6e72273267f7db371a9 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/descriptor_pool.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/duration.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/duration.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..69786cbd5531c6558c06866a850ddb6648f27160 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/duration.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/duration_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/duration_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e85f99f7248a66ef896712bb9b44659b3a7d063 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/duration_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/empty_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/empty_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f177ef05f118d5503680968edc038a0e1e8df12 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/empty_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/field_mask_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/field_mask_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1636c5a6f4375a399d5f78ca8949c6e5716c0302 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/field_mask_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/json_format.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/json_format.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fe5484059c8f94009e3938589108c753eaa4b47b Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/json_format.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/message.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/message.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d92456844248baf40e4ad260da093f258474618e Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/message.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/message_factory.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/message_factory.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cb58c4403f8f4beeb7d1bc71e6c9442823c6cd3f Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/message_factory.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/proto.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..00d6e076edc9eee5b320252c21d7fe39f4960fd0 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_builder.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_builder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..488356cd42ff7c49f465e49cd374f507160739cf Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_builder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_json.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_json.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..403fa87f45fdaeb53be0845441a8aed57db44cfa Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_json.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_text.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_text.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0f0dd7f1be874e679b79705a25857d96e09465d2 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/proto_text.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/reflection.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/reflection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..02e099008c58a34eef7cc00c0d14f26b7b1f3c1c Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/reflection.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/runtime_version.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/runtime_version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7fefd50f9ccaefc997d69f75ff6990022d2ae2f4 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/runtime_version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/service_reflection.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/service_reflection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f9315652d389d0dc4c9f0636f7a1264746e8102 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/service_reflection.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/source_context_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/source_context_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..532464c683e158f25dd058e6528175b222356272 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/source_context_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/struct_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/struct_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2d86390eded0e39d6a4985db4c7201393bc22e36 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/struct_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/symbol_database.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/symbol_database.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..313b5ffe936c83622a197ef67aadfd1c75176798 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/symbol_database.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/text_encoding.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/text_encoding.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8fde6088a2a17f6c4361d5dba50e63dba6741bd4 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/text_encoding.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/text_format.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/text_format.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..23b6ad59b8b02863d11750883dc20218824653a9 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/text_format.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/timestamp.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/timestamp.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5a8bd1f657ea0115919163aeb784192ae7719c2 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/timestamp.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/timestamp_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/timestamp_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..61f5f07a73dbd5028518789b47e891d46a2263cf Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/timestamp_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/type_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/type_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c9c69f14abbac9dfec2faadb8842c4fd9ea329ff Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/type_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/unknown_fields.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/unknown_fields.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f6b30d2389a829ba7c1b4e6772979f8a3c7b192b Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/unknown_fields.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/__pycache__/wrappers_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/__pycache__/wrappers_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..add6b4c50f3504fd22b991ec3b870233cfc9694d Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/__pycache__/wrappers_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/any.py b/lib/python3.12/site-packages/google/protobuf/any.py new file mode 100644 index 0000000000000000000000000000000000000000..acf37cf24c700f58a1a1d3080a9ddfd6403aa993 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/any.py @@ -0,0 +1,53 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains the Any helper APIs.""" + +from typing import Optional, TypeVar + +from google.protobuf import descriptor +from google.protobuf.message import Message + +from google.protobuf.any_pb2 import Any + + +_MessageT = TypeVar('_MessageT', bound=Message) + + +def pack( + msg: Message, + type_url_prefix: Optional[str] = 'type.googleapis.com/', + deterministic: Optional[bool] = None, +) -> Any: + any_msg = Any() + any_msg.Pack( + msg=msg, type_url_prefix=type_url_prefix, deterministic=deterministic + ) + return any_msg + + +def unpack(any_msg: Any, msg: Message) -> bool: + return any_msg.Unpack(msg=msg) + + +def unpack_as(any_msg: Any, message_type: type[_MessageT]) -> _MessageT: + unpacked = message_type() + if unpack(any_msg, unpacked): + return unpacked + else: + raise TypeError( + f'Attempted to unpack {type_name(any_msg)} to' + f' {message_type.__qualname__}' + ) + + +def type_name(any_msg: Any) -> str: + return any_msg.TypeName() + + +def is_type(any_msg: Any, des: descriptor.Descriptor) -> bool: + return any_msg.Is(des) diff --git a/lib/python3.12/site-packages/google/protobuf/any_pb2.py b/lib/python3.12/site-packages/google/protobuf/any_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..af20e337d9caf0090369bc995dfe1113c52cf94a --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/any_pb2.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/any.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/any.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19google/protobuf/any.proto\x12\x0fgoogle.protobuf\"6\n\x03\x41ny\x12\x19\n\x08type_url\x18\x01 \x01(\tR\x07typeUrl\x12\x14\n\x05value\x18\x02 \x01(\x0cR\x05valueBv\n\x13\x63om.google.protobufB\x08\x41nyProtoP\x01Z,google.golang.org/protobuf/types/known/anypb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.any_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\010AnyProtoP\001Z,google.golang.org/protobuf/types/known/anypb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_ANY']._serialized_start=46 + _globals['_ANY']._serialized_end=100 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/api_pb2.py b/lib/python3.12/site-packages/google/protobuf/api_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..f6190b7fc25ea488b04a981c676cf15cac2aac6a --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/api_pb2.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/api.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/api.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import source_context_pb2 as google_dot_protobuf_dot_source__context__pb2 +from google.protobuf import type_pb2 as google_dot_protobuf_dot_type__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x19google/protobuf/api.proto\x12\x0fgoogle.protobuf\x1a$google/protobuf/source_context.proto\x1a\x1agoogle/protobuf/type.proto\"\xdb\x02\n\x03\x41pi\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x31\n\x07methods\x18\x02 \x03(\x0b\x32\x17.google.protobuf.MethodR\x07methods\x12\x31\n\x07options\x18\x03 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x18\n\x07version\x18\x04 \x01(\tR\x07version\x12\x45\n\x0esource_context\x18\x05 \x01(\x0b\x32\x1e.google.protobuf.SourceContextR\rsourceContext\x12.\n\x06mixins\x18\x06 \x03(\x0b\x32\x16.google.protobuf.MixinR\x06mixins\x12/\n\x06syntax\x18\x07 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\x12\x18\n\x07\x65\x64ition\x18\x08 \x01(\tR\x07\x65\x64ition\"\xd4\x02\n\x06Method\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12(\n\x10request_type_url\x18\x02 \x01(\tR\x0erequestTypeUrl\x12+\n\x11request_streaming\x18\x03 \x01(\x08R\x10requestStreaming\x12*\n\x11response_type_url\x18\x04 \x01(\tR\x0fresponseTypeUrl\x12-\n\x12response_streaming\x18\x05 \x01(\x08R\x11responseStreaming\x12\x31\n\x07options\x18\x06 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x33\n\x06syntax\x18\x07 \x01(\x0e\x32\x17.google.protobuf.SyntaxB\x02\x18\x01R\x06syntax\x12\x1c\n\x07\x65\x64ition\x18\x08 \x01(\tB\x02\x18\x01R\x07\x65\x64ition\"/\n\x05Mixin\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x12\n\x04root\x18\x02 \x01(\tR\x04rootBv\n\x13\x63om.google.protobufB\x08\x41piProtoP\x01Z,google.golang.org/protobuf/types/known/apipb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.api_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\010ApiProtoP\001Z,google.golang.org/protobuf/types/known/apipb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_METHOD'].fields_by_name['syntax']._loaded_options = None + _globals['_METHOD'].fields_by_name['syntax']._serialized_options = b'\030\001' + _globals['_METHOD'].fields_by_name['edition']._loaded_options = None + _globals['_METHOD'].fields_by_name['edition']._serialized_options = b'\030\001' + _globals['_API']._serialized_start=113 + _globals['_API']._serialized_end=460 + _globals['_METHOD']._serialized_start=463 + _globals['_METHOD']._serialized_end=803 + _globals['_MIXIN']._serialized_start=805 + _globals['_MIXIN']._serialized_end=852 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/compiler/__init__.py b/lib/python3.12/site-packages/google/protobuf/compiler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/google/protobuf/compiler/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/compiler/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ed9bf920ad4e5128467177c81acd3ba8404c4a0 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/compiler/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/compiler/__pycache__/plugin_pb2.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/compiler/__pycache__/plugin_pb2.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa02856341fc0002f6c01ed50ffa14df1733db6f Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/compiler/__pycache__/plugin_pb2.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/compiler/plugin_pb2.py b/lib/python3.12/site-packages/google/protobuf/compiler/plugin_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..b9259b314d3cd84103db096cbe622d4cd44a72b3 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/compiler/plugin_pb2.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/compiler/plugin.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/compiler/plugin.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import descriptor_pb2 as google_dot_protobuf_dot_descriptor__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n%google/protobuf/compiler/plugin.proto\x12\x18google.protobuf.compiler\x1a google/protobuf/descriptor.proto\"c\n\x07Version\x12\x14\n\x05major\x18\x01 \x01(\x05R\x05major\x12\x14\n\x05minor\x18\x02 \x01(\x05R\x05minor\x12\x14\n\x05patch\x18\x03 \x01(\x05R\x05patch\x12\x16\n\x06suffix\x18\x04 \x01(\tR\x06suffix\"\xcf\x02\n\x14\x43odeGeneratorRequest\x12(\n\x10\x66ile_to_generate\x18\x01 \x03(\tR\x0e\x66ileToGenerate\x12\x1c\n\tparameter\x18\x02 \x01(\tR\tparameter\x12\x43\n\nproto_file\x18\x0f \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\tprotoFile\x12\\\n\x17source_file_descriptors\x18\x11 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x15sourceFileDescriptors\x12L\n\x10\x63ompiler_version\x18\x03 \x01(\x0b\x32!.google.protobuf.compiler.VersionR\x0f\x63ompilerVersion\"\x85\x04\n\x15\x43odeGeneratorResponse\x12\x14\n\x05\x65rror\x18\x01 \x01(\tR\x05\x65rror\x12-\n\x12supported_features\x18\x02 \x01(\x04R\x11supportedFeatures\x12\'\n\x0fminimum_edition\x18\x03 \x01(\x05R\x0eminimumEdition\x12\'\n\x0fmaximum_edition\x18\x04 \x01(\x05R\x0emaximumEdition\x12H\n\x04\x66ile\x18\x0f \x03(\x0b\x32\x34.google.protobuf.compiler.CodeGeneratorResponse.FileR\x04\x66ile\x1a\xb1\x01\n\x04\x46ile\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\'\n\x0finsertion_point\x18\x02 \x01(\tR\x0einsertionPoint\x12\x18\n\x07\x63ontent\x18\x0f \x01(\tR\x07\x63ontent\x12R\n\x13generated_code_info\x18\x10 \x01(\x0b\x32\".google.protobuf.GeneratedCodeInfoR\x11generatedCodeInfo\"W\n\x07\x46\x65\x61ture\x12\x10\n\x0c\x46\x45\x41TURE_NONE\x10\x00\x12\x1b\n\x17\x46\x45\x41TURE_PROTO3_OPTIONAL\x10\x01\x12\x1d\n\x19\x46\x45\x41TURE_SUPPORTS_EDITIONS\x10\x02\x42r\n\x1c\x63om.google.protobuf.compilerB\x0cPluginProtosZ)google.golang.org/protobuf/types/pluginpb\xaa\x02\x18Google.Protobuf.Compiler') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.compiler.plugin_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\034com.google.protobuf.compilerB\014PluginProtosZ)google.golang.org/protobuf/types/pluginpb\252\002\030Google.Protobuf.Compiler' + _globals['_VERSION']._serialized_start=101 + _globals['_VERSION']._serialized_end=200 + _globals['_CODEGENERATORREQUEST']._serialized_start=203 + _globals['_CODEGENERATORREQUEST']._serialized_end=538 + _globals['_CODEGENERATORRESPONSE']._serialized_start=541 + _globals['_CODEGENERATORRESPONSE']._serialized_end=1058 + _globals['_CODEGENERATORRESPONSE_FILE']._serialized_start=792 + _globals['_CODEGENERATORRESPONSE_FILE']._serialized_end=969 + _globals['_CODEGENERATORRESPONSE_FEATURE']._serialized_start=971 + _globals['_CODEGENERATORRESPONSE_FEATURE']._serialized_end=1058 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/descriptor.py b/lib/python3.12/site-packages/google/protobuf/descriptor.py new file mode 100644 index 0000000000000000000000000000000000000000..db46ae9f64831150899116d74ffb87a2510f9aa5 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/descriptor.py @@ -0,0 +1,1676 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Descriptors essentially contain exactly the information found in a .proto + +file, in types that make this information accessible in Python. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import abc +import binascii +import os +import threading +import warnings + +from google.protobuf.internal import api_implementation + +_USE_C_DESCRIPTORS = False +if api_implementation.Type() != 'python': + # pylint: disable=protected-access + _message = api_implementation._c_module + # TODO: Remove this import after fix api_implementation + if _message is None: + from google.protobuf.pyext import _message + _USE_C_DESCRIPTORS = True + + +class Error(Exception): + """Base error for this module.""" + + +class TypeTransformationError(Error): + """Error transforming between python proto type and corresponding C++ type.""" + + +if _USE_C_DESCRIPTORS: + # This metaclass allows to override the behavior of code like + # isinstance(my_descriptor, FieldDescriptor) + # and make it return True when the descriptor is an instance of the extension + # type written in C++. + class DescriptorMetaclass(type): + + def __instancecheck__(cls, obj): + if super(DescriptorMetaclass, cls).__instancecheck__(obj): + return True + if isinstance(obj, cls._C_DESCRIPTOR_CLASS): + return True + return False + +else: + # The standard metaclass; nothing changes. + DescriptorMetaclass = abc.ABCMeta + + +class _Lock(object): + """Wrapper class of threading.Lock(), which is allowed by 'with'.""" + + def __new__(cls): + self = object.__new__(cls) + self._lock = threading.Lock() # pylint: disable=protected-access + return self + + def __enter__(self): + self._lock.acquire() + + def __exit__(self, exc_type, exc_value, exc_tb): + self._lock.release() + + +_lock = threading.Lock() + + +def _Deprecated( + name, + alternative='get/find descriptors from generated code or query the descriptor_pool', +): + if _Deprecated.count > 0: + _Deprecated.count -= 1 + warnings.warn( + 'Call to deprecated %s, use %s instead.' % (name, alternative), + category=DeprecationWarning, + stacklevel=3, + ) + + +# These must match the values in descriptor.proto, but we can't use them +# directly because we sometimes need to reference them in feature helpers +# below *during* the build of descriptor.proto. +_FEATURESET_MESSAGE_ENCODING_DELIMITED = 2 +_FEATURESET_FIELD_PRESENCE_IMPLICIT = 2 +_FEATURESET_FIELD_PRESENCE_LEGACY_REQUIRED = 3 +_FEATURESET_REPEATED_FIELD_ENCODING_PACKED = 1 +_FEATURESET_ENUM_TYPE_CLOSED = 2 + +# Deprecated warnings will print 100 times at most which should be enough for +# users to notice and do not cause timeout. +_Deprecated.count = 100 + + +_internal_create_key = object() + + +class DescriptorBase(metaclass=DescriptorMetaclass): + """Descriptors base class. + + This class is the base of all descriptor classes. It provides common options + related functionality. + + Attributes: + has_options: True if the descriptor has non-default options. Usually it is + not necessary to read this -- just call GetOptions() which will happily + return the default instance. However, it's sometimes useful for + efficiency, and also useful inside the protobuf implementation to avoid + some bootstrapping issues. + file (FileDescriptor): Reference to file info. + """ + + if _USE_C_DESCRIPTORS: + # The class, or tuple of classes, that are considered as "virtual + # subclasses" of this descriptor class. + _C_DESCRIPTOR_CLASS = () + + def __init__(self, file, options, serialized_options, options_class_name): + """Initialize the descriptor given its options message and the name of the + + class of the options message. The name of the class is required in case + the options message is None and has to be created. + """ + self._features = None + self.file = file + self._original_options = options + # These two fields are duplicated as a compatibility shim for old gencode + # that resets them. In 26.x (cl/580304039) we renamed _options to, + # _loaded_options breaking backwards compatibility. + self._options = self._loaded_options = None + self._options_class_name = options_class_name + self._serialized_options = serialized_options + + # Does this descriptor have non-default options? + self.has_options = (self._original_options is not None) or ( + self._serialized_options is not None + ) + + @property + @abc.abstractmethod + def _parent(self): + pass + + def _InferLegacyFeatures(self, edition, options, features): + """Infers features from proto2/proto3 syntax so that editions logic can be used everywhere. + + Args: + edition: The edition to infer features for. + options: The options for this descriptor that are being processed. + features: The feature set object to modify with inferred features. + """ + pass + + def _GetFeatures(self): + if not self._features: + self._LazyLoadOptions() + return self._features + + def _ResolveFeatures(self, edition, raw_options): + """Resolves features from the raw options of this descriptor. + + Args: + edition: The edition to use for feature defaults. + raw_options: The options for this descriptor that are being processed. + + Returns: + A fully resolved feature set for making runtime decisions. + """ + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + + if self._parent: + features = descriptor_pb2.FeatureSet() + features.CopyFrom(self._parent._GetFeatures()) + else: + features = self.file.pool._CreateDefaultFeatures(edition) + unresolved = descriptor_pb2.FeatureSet() + unresolved.CopyFrom(raw_options.features) + self._InferLegacyFeatures(edition, raw_options, unresolved) + features.MergeFrom(unresolved) + + # Use the feature cache to reduce memory bloat. + return self.file.pool._InternFeatures(features) + + def _LazyLoadOptions(self): + """Lazily initializes descriptor options towards the end of the build.""" + if self._options and self._loaded_options == self._options: + # If neither has been reset by gencode, use the cache. + return + + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + + if not hasattr(descriptor_pb2, self._options_class_name): + raise RuntimeError( + 'Unknown options class name %s!' % self._options_class_name + ) + options_class = getattr(descriptor_pb2, self._options_class_name) + features = None + edition = self.file._edition + + if not self.has_options: + if not self._features: + features = self._ResolveFeatures( + descriptor_pb2.Edition.Value(edition), options_class() + ) + with _lock: + self._options = self._loaded_options = options_class() + if not self._features: + self._features = features + else: + if not self._serialized_options: + options = self._original_options + else: + options = _ParseOptions(options_class(), self._serialized_options) + + if not self._features: + features = self._ResolveFeatures( + descriptor_pb2.Edition.Value(edition), options + ) + with _lock: + self._options = self._loaded_options = options + if not self._features: + self._features = features + if options.HasField('features'): + options.ClearField('features') + if not options.SerializeToString(): + self._options = self._loaded_options = options_class() + self.has_options = False + + def GetOptions(self): + """Retrieves descriptor options. + + Returns: + The options set on this descriptor. + """ + # If either has been reset by gencode, reload options. + if not self._options or not self._loaded_options: + self._LazyLoadOptions() + return self._options + + +class _NestedDescriptorBase(DescriptorBase): + """Common class for descriptors that can be nested.""" + + def __init__( + self, + options, + options_class_name, + name, + full_name, + file, + containing_type, + serialized_start=None, + serialized_end=None, + serialized_options=None, + ): + """Constructor. + + Args: + options: Protocol message options or None to use default message options. + options_class_name (str): The class name of the above options. + name (str): Name of this protocol message type. + full_name (str): Fully-qualified name of this protocol message type, which + will include protocol "package" name and the name of any enclosing + types. + containing_type: if provided, this is a nested descriptor, with this + descriptor as parent, otherwise None. + serialized_start: The start index (inclusive) in block in the + file.serialized_pb that describes this descriptor. + serialized_end: The end index (exclusive) in block in the + file.serialized_pb that describes this descriptor. + serialized_options: Protocol message serialized options or None. + """ + super(_NestedDescriptorBase, self).__init__( + file, options, serialized_options, options_class_name + ) + + self.name = name + # TODO: Add function to calculate full_name instead of having it in + # memory? + self.full_name = full_name + self.containing_type = containing_type + + self._serialized_start = serialized_start + self._serialized_end = serialized_end + + def CopyToProto(self, proto): + """Copies this to the matching proto in descriptor_pb2. + + Args: + proto: An empty proto instance from descriptor_pb2. + + Raises: + Error: If self couldn't be serialized, due to to few constructor + arguments. + """ + if ( + self.file is not None + and self._serialized_start is not None + and self._serialized_end is not None + ): + proto.ParseFromString( + self.file.serialized_pb[self._serialized_start : self._serialized_end] + ) + else: + raise Error('Descriptor does not contain serialization.') + + +class Descriptor(_NestedDescriptorBase): + """Descriptor for a protocol message type. + + Attributes: + name (str): Name of this protocol message type. + full_name (str): Fully-qualified name of this protocol message type, which + will include protocol "package" name and the name of any enclosing + types. + containing_type (Descriptor): Reference to the descriptor of the type + containing us, or None if this is top-level. + fields (list[FieldDescriptor]): Field descriptors for all fields in this + type. + fields_by_number (dict(int, FieldDescriptor)): Same + :class:`FieldDescriptor` objects as in :attr:`fields`, but indexed by + "number" attribute in each FieldDescriptor. + fields_by_name (dict(str, FieldDescriptor)): Same :class:`FieldDescriptor` + objects as in :attr:`fields`, but indexed by "name" attribute in each + :class:`FieldDescriptor`. + nested_types (list[Descriptor]): Descriptor references for all protocol + message types nested within this one. + nested_types_by_name (dict(str, Descriptor)): Same Descriptor objects as + in :attr:`nested_types`, but indexed by "name" attribute in each + Descriptor. + enum_types (list[EnumDescriptor]): :class:`EnumDescriptor` references for + all enums contained within this type. + enum_types_by_name (dict(str, EnumDescriptor)): Same + :class:`EnumDescriptor` objects as in :attr:`enum_types`, but indexed by + "name" attribute in each EnumDescriptor. + enum_values_by_name (dict(str, EnumValueDescriptor)): Dict mapping from + enum value name to :class:`EnumValueDescriptor` for that value. + extensions (list[FieldDescriptor]): All extensions defined directly within + this message type (NOT within a nested type). + extensions_by_name (dict(str, FieldDescriptor)): Same FieldDescriptor + objects as :attr:`extensions`, but indexed by "name" attribute of each + FieldDescriptor. + is_extendable (bool): Does this type define any extension ranges? + oneofs (list[OneofDescriptor]): The list of descriptors for oneof fields + in this message. + oneofs_by_name (dict(str, OneofDescriptor)): Same objects as in + :attr:`oneofs`, but indexed by "name" attribute. + file (FileDescriptor): Reference to file descriptor. + is_map_entry: If the message type is a map entry. + """ + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.Descriptor + + def __new__( + cls, + name=None, + full_name=None, + filename=None, + containing_type=None, + fields=None, + nested_types=None, + enum_types=None, + extensions=None, + options=None, + serialized_options=None, + is_extendable=True, + extension_ranges=None, + oneofs=None, + file=None, # pylint: disable=redefined-builtin + serialized_start=None, + serialized_end=None, + syntax=None, + is_map_entry=False, + create_key=None, + ): + _message.Message._CheckCalledFromGeneratedFile() + return _message.default_pool.FindMessageTypeByName(full_name) + + # NOTE: The file argument redefining a builtin is nothing we can + # fix right now since we don't know how many clients already rely on the + # name of the argument. + def __init__( + self, + name, + full_name, + filename, + containing_type, + fields, + nested_types, + enum_types, + extensions, + options=None, + serialized_options=None, + is_extendable=True, + extension_ranges=None, + oneofs=None, + file=None, + serialized_start=None, + serialized_end=None, # pylint: disable=redefined-builtin + syntax=None, + is_map_entry=False, + create_key=None, + ): + """Arguments to __init__() are as described in the description + + of Descriptor fields above. + + Note that filename is an obsolete argument, that is not used anymore. + Please use file.name to access this as an attribute. + """ + if create_key is not _internal_create_key: + _Deprecated('create function Descriptor()') + + super(Descriptor, self).__init__( + options, + 'MessageOptions', + name, + full_name, + file, + containing_type, + serialized_start=serialized_start, + serialized_end=serialized_end, + serialized_options=serialized_options, + ) + + # We have fields in addition to fields_by_name and fields_by_number, + # so that: + # 1. Clients can index fields by "order in which they're listed." + # 2. Clients can easily iterate over all fields with the terse + # syntax: for f in descriptor.fields: ... + self.fields = fields + for field in self.fields: + field.containing_type = self + field.file = file + self.fields_by_number = dict((f.number, f) for f in fields) + self.fields_by_name = dict((f.name, f) for f in fields) + self._fields_by_camelcase_name = None + + self.nested_types = nested_types + for nested_type in nested_types: + nested_type.containing_type = self + self.nested_types_by_name = dict((t.name, t) for t in nested_types) + + self.enum_types = enum_types + for enum_type in self.enum_types: + enum_type.containing_type = self + self.enum_types_by_name = dict((t.name, t) for t in enum_types) + self.enum_values_by_name = dict( + (v.name, v) for t in enum_types for v in t.values + ) + + self.extensions = extensions + for extension in self.extensions: + extension.extension_scope = self + self.extensions_by_name = dict((f.name, f) for f in extensions) + self.is_extendable = is_extendable + self.extension_ranges = extension_ranges + self.oneofs = oneofs if oneofs is not None else [] + self.oneofs_by_name = dict((o.name, o) for o in self.oneofs) + for oneof in self.oneofs: + oneof.containing_type = self + oneof.file = file + self._is_map_entry = is_map_entry + + @property + def _parent(self): + return self.containing_type or self.file + + @property + def fields_by_camelcase_name(self): + """Same FieldDescriptor objects as in :attr:`fields`, but indexed by + + :attr:`FieldDescriptor.camelcase_name`. + """ + if self._fields_by_camelcase_name is None: + self._fields_by_camelcase_name = dict( + (f.camelcase_name, f) for f in self.fields + ) + return self._fields_by_camelcase_name + + def EnumValueName(self, enum, value): + """Returns the string name of an enum value. + + This is just a small helper method to simplify a common operation. + + Args: + enum: string name of the Enum. + value: int, value of the enum. + + Returns: + string name of the enum value. + + Raises: + KeyError if either the Enum doesn't exist or the value is not a valid + value for the enum. + """ + return self.enum_types_by_name[enum].values_by_number[value].name + + def CopyToProto(self, proto): + """Copies this to a descriptor_pb2.DescriptorProto. + + Args: + proto: An empty descriptor_pb2.DescriptorProto. + """ + # This function is overridden to give a better doc comment. + super(Descriptor, self).CopyToProto(proto) + + +# TODO: We should have aggressive checking here, +# for example: +# * If you specify a repeated field, you should not be allowed +# to specify a default value. +# * [Other examples here as needed]. +# +# TODO: for this and other *Descriptor classes, we +# might also want to lock things down aggressively (e.g., +# prevent clients from setting the attributes). Having +# stronger invariants here in general will reduce the number +# of runtime checks we must do in reflection.py... +class FieldDescriptor(DescriptorBase): + """Descriptor for a single field in a .proto file. + + Attributes: + name (str): Name of this field, exactly as it appears in .proto. + full_name (str): Name of this field, including containing scope. This is + particularly relevant for extensions. + index (int): Dense, 0-indexed index giving the order that this field + textually appears within its message in the .proto file. + number (int): Tag number declared for this field in the .proto file. + type (int): (One of the TYPE_* constants below) Declared type. + cpp_type (int): (One of the CPPTYPE_* constants below) C++ type used to + represent this field. + label (int): (One of the LABEL_* constants below) Tells whether this field + is optional, required, or repeated. + has_default_value (bool): True if this field has a default value defined, + otherwise false. + default_value (Varies): Default value of this field. Only meaningful for + non-repeated scalar fields. Repeated fields should always set this to [], + and non-repeated composite fields should always set this to None. + containing_type (Descriptor): Descriptor of the protocol message type that + contains this field. Set by the Descriptor constructor if we're passed + into one. Somewhat confusingly, for extension fields, this is the + descriptor of the EXTENDED message, not the descriptor of the message + containing this field. (See is_extension and extension_scope below). + message_type (Descriptor): If a composite field, a descriptor of the message + type contained in this field. Otherwise, this is None. + enum_type (EnumDescriptor): If this field contains an enum, a descriptor of + that enum. Otherwise, this is None. + is_extension: True iff this describes an extension field. + extension_scope (Descriptor): Only meaningful if is_extension is True. Gives + the message that immediately contains this extension field. Will be None + iff we're a top-level (file-level) extension field. + options (descriptor_pb2.FieldOptions): Protocol message field options or + None to use default field options. + containing_oneof (OneofDescriptor): If the field is a member of a oneof + union, contains its descriptor. Otherwise, None. + file (FileDescriptor): Reference to file descriptor. + """ + + # Must be consistent with C++ FieldDescriptor::Type enum in + # descriptor.h. + # + # TODO: Find a way to eliminate this repetition. + TYPE_DOUBLE = 1 + TYPE_FLOAT = 2 + TYPE_INT64 = 3 + TYPE_UINT64 = 4 + TYPE_INT32 = 5 + TYPE_FIXED64 = 6 + TYPE_FIXED32 = 7 + TYPE_BOOL = 8 + TYPE_STRING = 9 + TYPE_GROUP = 10 + TYPE_MESSAGE = 11 + TYPE_BYTES = 12 + TYPE_UINT32 = 13 + TYPE_ENUM = 14 + TYPE_SFIXED32 = 15 + TYPE_SFIXED64 = 16 + TYPE_SINT32 = 17 + TYPE_SINT64 = 18 + MAX_TYPE = 18 + + # Must be consistent with C++ FieldDescriptor::CppType enum in + # descriptor.h. + # + # TODO: Find a way to eliminate this repetition. + CPPTYPE_INT32 = 1 + CPPTYPE_INT64 = 2 + CPPTYPE_UINT32 = 3 + CPPTYPE_UINT64 = 4 + CPPTYPE_DOUBLE = 5 + CPPTYPE_FLOAT = 6 + CPPTYPE_BOOL = 7 + CPPTYPE_ENUM = 8 + CPPTYPE_STRING = 9 + CPPTYPE_MESSAGE = 10 + MAX_CPPTYPE = 10 + + _PYTHON_TO_CPP_PROTO_TYPE_MAP = { + TYPE_DOUBLE: CPPTYPE_DOUBLE, + TYPE_FLOAT: CPPTYPE_FLOAT, + TYPE_ENUM: CPPTYPE_ENUM, + TYPE_INT64: CPPTYPE_INT64, + TYPE_SINT64: CPPTYPE_INT64, + TYPE_SFIXED64: CPPTYPE_INT64, + TYPE_UINT64: CPPTYPE_UINT64, + TYPE_FIXED64: CPPTYPE_UINT64, + TYPE_INT32: CPPTYPE_INT32, + TYPE_SFIXED32: CPPTYPE_INT32, + TYPE_SINT32: CPPTYPE_INT32, + TYPE_UINT32: CPPTYPE_UINT32, + TYPE_FIXED32: CPPTYPE_UINT32, + TYPE_BYTES: CPPTYPE_STRING, + TYPE_STRING: CPPTYPE_STRING, + TYPE_BOOL: CPPTYPE_BOOL, + TYPE_MESSAGE: CPPTYPE_MESSAGE, + TYPE_GROUP: CPPTYPE_MESSAGE, + } + + # Must be consistent with C++ FieldDescriptor::Label enum in + # descriptor.h. + # + # TODO: Find a way to eliminate this repetition. + LABEL_OPTIONAL = 1 + LABEL_REQUIRED = 2 + LABEL_REPEATED = 3 + MAX_LABEL = 3 + + # Must be consistent with C++ constants kMaxNumber, kFirstReservedNumber, + # and kLastReservedNumber in descriptor.h + MAX_FIELD_NUMBER = (1 << 29) - 1 + FIRST_RESERVED_FIELD_NUMBER = 19000 + LAST_RESERVED_FIELD_NUMBER = 19999 + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.FieldDescriptor + + def __new__( + cls, + name, + full_name, + index, + number, + type, + cpp_type, + label, + default_value, + message_type, + enum_type, + containing_type, + is_extension, + extension_scope, + options=None, + serialized_options=None, + has_default_value=True, + containing_oneof=None, + json_name=None, + file=None, + create_key=None, + ): # pylint: disable=redefined-builtin + _message.Message._CheckCalledFromGeneratedFile() + if is_extension: + return _message.default_pool.FindExtensionByName(full_name) + else: + return _message.default_pool.FindFieldByName(full_name) + + def __init__( + self, + name, + full_name, + index, + number, + type, + cpp_type, + label, + default_value, + message_type, + enum_type, + containing_type, + is_extension, + extension_scope, + options=None, + serialized_options=None, + has_default_value=True, + containing_oneof=None, + json_name=None, + file=None, + create_key=None, + ): # pylint: disable=redefined-builtin + """The arguments are as described in the description of FieldDescriptor + + attributes above. + + Note that containing_type may be None, and may be set later if necessary + (to deal with circular references between message types, for example). + Likewise for extension_scope. + """ + if create_key is not _internal_create_key: + _Deprecated('create function FieldDescriptor()') + + super(FieldDescriptor, self).__init__( + file, options, serialized_options, 'FieldOptions' + ) + self.name = name + self.full_name = full_name + self._camelcase_name = None + if json_name is None: + self.json_name = _ToJsonName(name) + else: + self.json_name = json_name + self.index = index + self.number = number + self._type = type + self.cpp_type = cpp_type + self._label = label + self.has_default_value = has_default_value + self.default_value = default_value + self.containing_type = containing_type + self.message_type = message_type + self.enum_type = enum_type + self.is_extension = is_extension + self.extension_scope = extension_scope + self.containing_oneof = containing_oneof + if api_implementation.Type() == 'python': + self._cdescriptor = None + else: + if is_extension: + self._cdescriptor = _message.default_pool.FindExtensionByName(full_name) + else: + self._cdescriptor = _message.default_pool.FindFieldByName(full_name) + + @property + def _parent(self): + if self.containing_oneof: + return self.containing_oneof + if self.is_extension: + return self.extension_scope or self.file + return self.containing_type + + def _InferLegacyFeatures(self, edition, options, features): + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + + if edition >= descriptor_pb2.Edition.EDITION_2023: + return + + if self._label == FieldDescriptor.LABEL_REQUIRED: + features.field_presence = ( + descriptor_pb2.FeatureSet.FieldPresence.LEGACY_REQUIRED + ) + + if self._type == FieldDescriptor.TYPE_GROUP: + features.message_encoding = ( + descriptor_pb2.FeatureSet.MessageEncoding.DELIMITED + ) + + if options.HasField('packed'): + features.repeated_field_encoding = ( + descriptor_pb2.FeatureSet.RepeatedFieldEncoding.PACKED + if options.packed + else descriptor_pb2.FeatureSet.RepeatedFieldEncoding.EXPANDED + ) + + @property + def type(self): + if ( + self._GetFeatures().message_encoding + == _FEATURESET_MESSAGE_ENCODING_DELIMITED + and self.message_type + and not self.message_type.GetOptions().map_entry + and not self.containing_type.GetOptions().map_entry + ): + return FieldDescriptor.TYPE_GROUP + return self._type + + @type.setter + def type(self, val): + self._type = val + + @property + def label(self): + _Deprecated('label property', 'is_required or is_repeated properties') + + if ( + self._GetFeatures().field_presence + == _FEATURESET_FIELD_PRESENCE_LEGACY_REQUIRED + ): + return FieldDescriptor.LABEL_REQUIRED + return self._label + + @property + def is_required(self): + """Returns if the field is required.""" + return ( + self._GetFeatures().field_presence + == _FEATURESET_FIELD_PRESENCE_LEGACY_REQUIRED + ) + + @property + def is_repeated(self): + """Returns if the field is repeated.""" + return self._label == FieldDescriptor.LABEL_REPEATED + + @property + def camelcase_name(self): + """Camelcase name of this field. + + Returns: + str: the name in CamelCase. + """ + if self._camelcase_name is None: + self._camelcase_name = _ToCamelCase(self.name) + return self._camelcase_name + + @property + def has_presence(self): + """Whether the field distinguishes between unpopulated and default values. + + Raises: + RuntimeError: singular field that is not linked with message nor file. + """ + if self.is_repeated: + return False + if ( + self.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE + or self.is_extension + or self.containing_oneof + ): + return True + + return ( + self._GetFeatures().field_presence + != _FEATURESET_FIELD_PRESENCE_IMPLICIT + ) + + @property + def is_packed(self): + """Returns if the field is packed.""" + if not self.is_repeated: + return False + field_type = self.type + if ( + field_type == FieldDescriptor.TYPE_STRING + or field_type == FieldDescriptor.TYPE_GROUP + or field_type == FieldDescriptor.TYPE_MESSAGE + or field_type == FieldDescriptor.TYPE_BYTES + ): + return False + + return ( + self._GetFeatures().repeated_field_encoding + == _FEATURESET_REPEATED_FIELD_ENCODING_PACKED + ) + + @staticmethod + def ProtoTypeToCppProtoType(proto_type): + """Converts from a Python proto type to a C++ Proto Type. + + The Python ProtocolBuffer classes specify both the 'Python' datatype and the + 'C++' datatype - and they're not the same. This helper method should + translate from one to another. + + Args: + proto_type: the Python proto type (descriptor.FieldDescriptor.TYPE_*) + + Returns: + int: descriptor.FieldDescriptor.CPPTYPE_*, the C++ type. + Raises: + TypeTransformationError: when the Python proto type isn't known. + """ + try: + return FieldDescriptor._PYTHON_TO_CPP_PROTO_TYPE_MAP[proto_type] + except KeyError: + raise TypeTransformationError('Unknown proto_type: %s' % proto_type) + + +class EnumDescriptor(_NestedDescriptorBase): + """Descriptor for an enum defined in a .proto file. + + Attributes: + name (str): Name of the enum type. + full_name (str): Full name of the type, including package name and any + enclosing type(s). + values (list[EnumValueDescriptor]): List of the values in this enum. + values_by_name (dict(str, EnumValueDescriptor)): Same as :attr:`values`, but + indexed by the "name" field of each EnumValueDescriptor. + values_by_number (dict(int, EnumValueDescriptor)): Same as :attr:`values`, + but indexed by the "number" field of each EnumValueDescriptor. + containing_type (Descriptor): Descriptor of the immediate containing type of + this enum, or None if this is an enum defined at the top level in a .proto + file. Set by Descriptor's constructor if we're passed into one. + file (FileDescriptor): Reference to file descriptor. + options (descriptor_pb2.EnumOptions): Enum options message or None to use + default enum options. + """ + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.EnumDescriptor + + def __new__( + cls, + name, + full_name, + filename, + values, + containing_type=None, + options=None, + serialized_options=None, + file=None, # pylint: disable=redefined-builtin + serialized_start=None, + serialized_end=None, + create_key=None, + ): + _message.Message._CheckCalledFromGeneratedFile() + return _message.default_pool.FindEnumTypeByName(full_name) + + def __init__( + self, + name, + full_name, + filename, + values, + containing_type=None, + options=None, + serialized_options=None, + file=None, # pylint: disable=redefined-builtin + serialized_start=None, + serialized_end=None, + create_key=None, + ): + """Arguments are as described in the attribute description above. + + Note that filename is an obsolete argument, that is not used anymore. + Please use file.name to access this as an attribute. + """ + if create_key is not _internal_create_key: + _Deprecated('create function EnumDescriptor()') + + super(EnumDescriptor, self).__init__( + options, + 'EnumOptions', + name, + full_name, + file, + containing_type, + serialized_start=serialized_start, + serialized_end=serialized_end, + serialized_options=serialized_options, + ) + + self.values = values + for value in self.values: + value.file = file + value.type = self + self.values_by_name = dict((v.name, v) for v in values) + # Values are reversed to ensure that the first alias is retained. + self.values_by_number = dict((v.number, v) for v in reversed(values)) + + @property + def _parent(self): + return self.containing_type or self.file + + @property + def is_closed(self): + """Returns true whether this is a "closed" enum. + + This means that it: + - Has a fixed set of values, rather than being equivalent to an int32. + - Encountering values not in this set causes them to be treated as unknown + fields. + - The first value (i.e., the default) may be nonzero. + + WARNING: Some runtimes currently have a quirk where non-closed enums are + treated as closed when used as the type of fields defined in a + `syntax = proto2;` file. This quirk is not present in all runtimes; as of + writing, we know that: + + - C++, Java, and C++-based Python share this quirk. + - UPB and UPB-based Python do not. + - PHP and Ruby treat all enums as open regardless of declaration. + + Care should be taken when using this function to respect the target + runtime's enum handling quirks. + """ + return self._GetFeatures().enum_type == _FEATURESET_ENUM_TYPE_CLOSED + + def CopyToProto(self, proto): + """Copies this to a descriptor_pb2.EnumDescriptorProto. + + Args: + proto (descriptor_pb2.EnumDescriptorProto): An empty descriptor proto. + """ + # This function is overridden to give a better doc comment. + super(EnumDescriptor, self).CopyToProto(proto) + + +class EnumValueDescriptor(DescriptorBase): + """Descriptor for a single value within an enum. + + Attributes: + name (str): Name of this value. + index (int): Dense, 0-indexed index giving the order that this value appears + textually within its enum in the .proto file. + number (int): Actual number assigned to this enum value. + type (EnumDescriptor): :class:`EnumDescriptor` to which this value belongs. + Set by :class:`EnumDescriptor`'s constructor if we're passed into one. + options (descriptor_pb2.EnumValueOptions): Enum value options message or + None to use default enum value options options. + """ + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.EnumValueDescriptor + + def __new__( + cls, + name, + index, + number, + type=None, # pylint: disable=redefined-builtin + options=None, + serialized_options=None, + create_key=None, + ): + _message.Message._CheckCalledFromGeneratedFile() + # There is no way we can build a complete EnumValueDescriptor with the + # given parameters (the name of the Enum is not known, for example). + # Fortunately generated files just pass it to the EnumDescriptor() + # constructor, which will ignore it, so returning None is good enough. + return None + + def __init__( + self, + name, + index, + number, + type=None, # pylint: disable=redefined-builtin + options=None, + serialized_options=None, + create_key=None, + ): + """Arguments are as described in the attribute description above.""" + if create_key is not _internal_create_key: + _Deprecated('create function EnumValueDescriptor()') + + super(EnumValueDescriptor, self).__init__( + type.file if type else None, + options, + serialized_options, + 'EnumValueOptions', + ) + self.name = name + self.index = index + self.number = number + self.type = type + + @property + def _parent(self): + return self.type + + +class OneofDescriptor(DescriptorBase): + """Descriptor for a oneof field. + + Attributes: + name (str): Name of the oneof field. + full_name (str): Full name of the oneof field, including package name. + index (int): 0-based index giving the order of the oneof field inside its + containing type. + containing_type (Descriptor): :class:`Descriptor` of the protocol message + type that contains this field. Set by the :class:`Descriptor` constructor + if we're passed into one. + fields (list[FieldDescriptor]): The list of field descriptors this oneof can + contain. + """ + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.OneofDescriptor + + def __new__( + cls, + name, + full_name, + index, + containing_type, + fields, + options=None, + serialized_options=None, + create_key=None, + ): + _message.Message._CheckCalledFromGeneratedFile() + return _message.default_pool.FindOneofByName(full_name) + + def __init__( + self, + name, + full_name, + index, + containing_type, + fields, + options=None, + serialized_options=None, + create_key=None, + ): + """Arguments are as described in the attribute description above.""" + if create_key is not _internal_create_key: + _Deprecated('create function OneofDescriptor()') + + super(OneofDescriptor, self).__init__( + containing_type.file if containing_type else None, + options, + serialized_options, + 'OneofOptions', + ) + self.name = name + self.full_name = full_name + self.index = index + self.containing_type = containing_type + self.fields = fields + + @property + def _parent(self): + return self.containing_type + + +class ServiceDescriptor(_NestedDescriptorBase): + """Descriptor for a service. + + Attributes: + name (str): Name of the service. + full_name (str): Full name of the service, including package name. + index (int): 0-indexed index giving the order that this services definition + appears within the .proto file. + methods (list[MethodDescriptor]): List of methods provided by this service. + methods_by_name (dict(str, MethodDescriptor)): Same + :class:`MethodDescriptor` objects as in :attr:`methods_by_name`, but + indexed by "name" attribute in each :class:`MethodDescriptor`. + options (descriptor_pb2.ServiceOptions): Service options message or None to + use default service options. + file (FileDescriptor): Reference to file info. + """ + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.ServiceDescriptor + + def __new__( + cls, + name=None, + full_name=None, + index=None, + methods=None, + options=None, + serialized_options=None, + file=None, # pylint: disable=redefined-builtin + serialized_start=None, + serialized_end=None, + create_key=None, + ): + _message.Message._CheckCalledFromGeneratedFile() # pylint: disable=protected-access + return _message.default_pool.FindServiceByName(full_name) + + def __init__( + self, + name, + full_name, + index, + methods, + options=None, + serialized_options=None, + file=None, # pylint: disable=redefined-builtin + serialized_start=None, + serialized_end=None, + create_key=None, + ): + if create_key is not _internal_create_key: + _Deprecated('create function ServiceDescriptor()') + + super(ServiceDescriptor, self).__init__( + options, + 'ServiceOptions', + name, + full_name, + file, + None, + serialized_start=serialized_start, + serialized_end=serialized_end, + serialized_options=serialized_options, + ) + self.index = index + self.methods = methods + self.methods_by_name = dict((m.name, m) for m in methods) + # Set the containing service for each method in this service. + for method in self.methods: + method.file = self.file + method.containing_service = self + + @property + def _parent(self): + return self.file + + def FindMethodByName(self, name): + """Searches for the specified method, and returns its descriptor. + + Args: + name (str): Name of the method. + + Returns: + MethodDescriptor: The descriptor for the requested method. + + Raises: + KeyError: if the method cannot be found in the service. + """ + return self.methods_by_name[name] + + def CopyToProto(self, proto): + """Copies this to a descriptor_pb2.ServiceDescriptorProto. + + Args: + proto (descriptor_pb2.ServiceDescriptorProto): An empty descriptor proto. + """ + # This function is overridden to give a better doc comment. + super(ServiceDescriptor, self).CopyToProto(proto) + + +class MethodDescriptor(DescriptorBase): + """Descriptor for a method in a service. + + Attributes: + name (str): Name of the method within the service. + full_name (str): Full name of method. + index (int): 0-indexed index of the method inside the service. + containing_service (ServiceDescriptor): The service that contains this + method. + input_type (Descriptor): The descriptor of the message that this method + accepts. + output_type (Descriptor): The descriptor of the message that this method + returns. + client_streaming (bool): Whether this method uses client streaming. + server_streaming (bool): Whether this method uses server streaming. + options (descriptor_pb2.MethodOptions or None): Method options message, or + None to use default method options. + """ + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.MethodDescriptor + + def __new__( + cls, + name, + full_name, + index, + containing_service, + input_type, + output_type, + client_streaming=False, + server_streaming=False, + options=None, + serialized_options=None, + create_key=None, + ): + _message.Message._CheckCalledFromGeneratedFile() # pylint: disable=protected-access + return _message.default_pool.FindMethodByName(full_name) + + def __init__( + self, + name, + full_name, + index, + containing_service, + input_type, + output_type, + client_streaming=False, + server_streaming=False, + options=None, + serialized_options=None, + create_key=None, + ): + """The arguments are as described in the description of MethodDescriptor + + attributes above. + + Note that containing_service may be None, and may be set later if necessary. + """ + if create_key is not _internal_create_key: + _Deprecated('create function MethodDescriptor()') + + super(MethodDescriptor, self).__init__( + containing_service.file if containing_service else None, + options, + serialized_options, + 'MethodOptions', + ) + self.name = name + self.full_name = full_name + self.index = index + self.containing_service = containing_service + self.input_type = input_type + self.output_type = output_type + self.client_streaming = client_streaming + self.server_streaming = server_streaming + + @property + def _parent(self): + return self.containing_service + + def CopyToProto(self, proto): + """Copies this to a descriptor_pb2.MethodDescriptorProto. + + Args: + proto (descriptor_pb2.MethodDescriptorProto): An empty descriptor proto. + + Raises: + Error: If self couldn't be serialized, due to too few constructor + arguments. + """ + if self.containing_service is not None: + from google.protobuf import descriptor_pb2 + + service_proto = descriptor_pb2.ServiceDescriptorProto() + self.containing_service.CopyToProto(service_proto) + proto.CopyFrom(service_proto.method[self.index]) + else: + raise Error('Descriptor does not contain a service.') + + +class FileDescriptor(DescriptorBase): + """Descriptor for a file. Mimics the descriptor_pb2.FileDescriptorProto. + + Note that :attr:`enum_types_by_name`, :attr:`extensions_by_name`, and + :attr:`dependencies` fields are only set by the + :py:mod:`google.protobuf.message_factory` module, and not by the generated + proto code. + + Attributes: + name (str): Name of file, relative to root of source tree. + package (str): Name of the package + edition (Edition): Enum value indicating edition of the file + serialized_pb (bytes): Byte string of serialized + :class:`descriptor_pb2.FileDescriptorProto`. + dependencies (list[FileDescriptor]): List of other :class:`FileDescriptor` + objects this :class:`FileDescriptor` depends on. + public_dependencies (list[FileDescriptor]): A subset of + :attr:`dependencies`, which were declared as "public". + message_types_by_name (dict(str, Descriptor)): Mapping from message names to + their :class:`Descriptor`. + enum_types_by_name (dict(str, EnumDescriptor)): Mapping from enum names to + their :class:`EnumDescriptor`. + extensions_by_name (dict(str, FieldDescriptor)): Mapping from extension + names declared at file scope to their :class:`FieldDescriptor`. + services_by_name (dict(str, ServiceDescriptor)): Mapping from services' + names to their :class:`ServiceDescriptor`. + pool (DescriptorPool): The pool this descriptor belongs to. When not passed + to the constructor, the global default pool is used. + """ + + if _USE_C_DESCRIPTORS: + _C_DESCRIPTOR_CLASS = _message.FileDescriptor + + def __new__( + cls, + name, + package, + options=None, + serialized_options=None, + serialized_pb=None, + dependencies=None, + public_dependencies=None, + syntax=None, + edition=None, + pool=None, + create_key=None, + ): + # FileDescriptor() is called from various places, not only from generated + # files, to register dynamic proto files and messages. + # pylint: disable=g-explicit-bool-comparison + if serialized_pb: + return _message.default_pool.AddSerializedFile(serialized_pb) + else: + return super(FileDescriptor, cls).__new__(cls) + + def __init__( + self, + name, + package, + options=None, + serialized_options=None, + serialized_pb=None, + dependencies=None, + public_dependencies=None, + syntax=None, + edition=None, + pool=None, + create_key=None, + ): + """Constructor.""" + if create_key is not _internal_create_key: + _Deprecated('create function FileDescriptor()') + + super(FileDescriptor, self).__init__( + self, options, serialized_options, 'FileOptions' + ) + + if edition and edition != 'EDITION_UNKNOWN': + self._edition = edition + elif syntax == 'proto3': + self._edition = 'EDITION_PROTO3' + else: + self._edition = 'EDITION_PROTO2' + + if pool is None: + from google.protobuf import descriptor_pool + + pool = descriptor_pool.Default() + self.pool = pool + self.message_types_by_name = {} + self.name = name + self.package = package + self.serialized_pb = serialized_pb + + self.enum_types_by_name = {} + self.extensions_by_name = {} + self.services_by_name = {} + self.dependencies = dependencies or [] + self.public_dependencies = public_dependencies or [] + + def CopyToProto(self, proto): + """Copies this to a descriptor_pb2.FileDescriptorProto. + + Args: + proto: An empty descriptor_pb2.FileDescriptorProto. + """ + proto.ParseFromString(self.serialized_pb) + + @property + def _parent(self): + return None + + +def _ParseOptions(message, string): + """Parses serialized options. + + This helper function is used to parse serialized options in generated + proto2 files. It must not be used outside proto2. + """ + message.ParseFromString(string) + return message + + +def _ToCamelCase(name): + """Converts name to camel-case and returns it.""" + capitalize_next = False + result = [] + + for c in name: + if c == '_': + if result: + capitalize_next = True + elif capitalize_next: + result.append(c.upper()) + capitalize_next = False + else: + result += c + + # Lower-case the first letter. + if result and result[0].isupper(): + result[0] = result[0].lower() + return ''.join(result) + + +def _OptionsOrNone(descriptor_proto): + """Returns the value of the field `options`, or None if it is not set.""" + if descriptor_proto.HasField('options'): + return descriptor_proto.options + else: + return None + + +def _ToJsonName(name): + """Converts name to Json name and returns it.""" + capitalize_next = False + result = [] + + for c in name: + if c == '_': + capitalize_next = True + elif capitalize_next: + result.append(c.upper()) + capitalize_next = False + else: + result += c + + return ''.join(result) + + +def MakeDescriptor( + desc_proto, + package='', + build_file_if_cpp=True, + syntax=None, + edition=None, + file_desc=None, +): + """Make a protobuf Descriptor given a DescriptorProto protobuf. + + Handles nested descriptors. Note that this is limited to the scope of defining + a message inside of another message. Composite fields can currently only be + resolved if the message is defined in the same scope as the field. + + Args: + desc_proto: The descriptor_pb2.DescriptorProto protobuf message. + package: Optional package name for the new message Descriptor (string). + build_file_if_cpp: Update the C++ descriptor pool if api matches. Set to + False on recursion, so no duplicates are created. + syntax: The syntax/semantics that should be used. Set to "proto3" to get + proto3 field presence semantics. + edition: The edition that should be used if syntax is "edition". + file_desc: A FileDescriptor to place this descriptor into. + + Returns: + A Descriptor for protobuf messages. + """ + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + + # Generate a random name for this proto file to prevent conflicts with any + # imported ones. We need to specify a file name so the descriptor pool + # accepts our FileDescriptorProto, but it is not important what that file + # name is actually set to. + proto_name = binascii.hexlify(os.urandom(16)).decode('ascii') + + if package: + file_name = os.path.join(package.replace('.', '/'), proto_name + '.proto') + else: + file_name = proto_name + '.proto' + + if api_implementation.Type() != 'python' and build_file_if_cpp: + # The C++ implementation requires all descriptors to be backed by the same + # definition in the C++ descriptor pool. To do this, we build a + # FileDescriptorProto with the same definition as this descriptor and build + # it into the pool. + file_descriptor_proto = descriptor_pb2.FileDescriptorProto() + file_descriptor_proto.message_type.add().MergeFrom(desc_proto) + + if package: + file_descriptor_proto.package = package + file_descriptor_proto.name = file_name + + _message.default_pool.Add(file_descriptor_proto) + result = _message.default_pool.FindFileByName(file_descriptor_proto.name) + + if _USE_C_DESCRIPTORS: + return result.message_types_by_name[desc_proto.name] + + if file_desc is None: + file_desc = FileDescriptor( + pool=None, + name=file_name, + package=package, + syntax=syntax, + edition=edition, + options=None, + serialized_pb='', + dependencies=[], + public_dependencies=[], + create_key=_internal_create_key, + ) + full_message_name = [desc_proto.name] + if package: + full_message_name.insert(0, package) + + # Create Descriptors for enum types + enum_types = {} + for enum_proto in desc_proto.enum_type: + full_name = '.'.join(full_message_name + [enum_proto.name]) + enum_desc = EnumDescriptor( + enum_proto.name, + full_name, + None, + [ + EnumValueDescriptor( + enum_val.name, + ii, + enum_val.number, + create_key=_internal_create_key, + ) + for ii, enum_val in enumerate(enum_proto.value) + ], + file=file_desc, + create_key=_internal_create_key, + ) + enum_types[full_name] = enum_desc + + # Create Descriptors for nested types + nested_types = {} + for nested_proto in desc_proto.nested_type: + full_name = '.'.join(full_message_name + [nested_proto.name]) + # Nested types are just those defined inside of the message, not all types + # used by fields in the message, so no loops are possible here. + nested_desc = MakeDescriptor( + nested_proto, + package='.'.join(full_message_name), + build_file_if_cpp=False, + syntax=syntax, + edition=edition, + file_desc=file_desc, + ) + nested_types[full_name] = nested_desc + + fields = [] + for field_proto in desc_proto.field: + full_name = '.'.join(full_message_name + [field_proto.name]) + enum_desc = None + nested_desc = None + if field_proto.json_name: + json_name = field_proto.json_name + else: + json_name = None + if field_proto.HasField('type_name'): + type_name = field_proto.type_name + full_type_name = '.'.join( + full_message_name + [type_name[type_name.rfind('.') + 1 :]] + ) + if full_type_name in nested_types: + nested_desc = nested_types[full_type_name] + elif full_type_name in enum_types: + enum_desc = enum_types[full_type_name] + # Else type_name references a non-local type, which isn't implemented + field = FieldDescriptor( + field_proto.name, + full_name, + field_proto.number - 1, + field_proto.number, + field_proto.type, + FieldDescriptor.ProtoTypeToCppProtoType(field_proto.type), + field_proto.label, + None, + nested_desc, + enum_desc, + None, + False, + None, + options=_OptionsOrNone(field_proto), + has_default_value=False, + json_name=json_name, + file=file_desc, + create_key=_internal_create_key, + ) + fields.append(field) + + desc_name = '.'.join(full_message_name) + return Descriptor( + desc_proto.name, + desc_name, + None, + None, + fields, + list(nested_types.values()), + list(enum_types.values()), + [], + options=_OptionsOrNone(desc_proto), + file=file_desc, + create_key=_internal_create_key, + ) diff --git a/lib/python3.12/site-packages/google/protobuf/descriptor_database.py b/lib/python3.12/site-packages/google/protobuf/descriptor_database.py new file mode 100644 index 0000000000000000000000000000000000000000..eeb8599372827ea36346e1cb579cf6066d1fb674 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/descriptor_database.py @@ -0,0 +1,172 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Provides a container for DescriptorProtos.""" + +__author__ = 'matthewtoia@google.com (Matt Toia)' + +import warnings + + +class Error(Exception): + pass + + +class DescriptorDatabaseConflictingDefinitionError(Error): + """Raised when a proto is added with the same name & different descriptor.""" + + +class DescriptorDatabase(object): + """A container accepting FileDescriptorProtos and maps DescriptorProtos.""" + + def __init__(self): + self._file_desc_protos_by_file = {} + self._file_desc_protos_by_symbol = {} + + def Add(self, file_desc_proto): + """Adds the FileDescriptorProto and its types to this database. + + Args: + file_desc_proto: The FileDescriptorProto to add. + Raises: + DescriptorDatabaseConflictingDefinitionError: if an attempt is made to + add a proto with the same name but different definition than an + existing proto in the database. + """ + proto_name = file_desc_proto.name + if proto_name not in self._file_desc_protos_by_file: + self._file_desc_protos_by_file[proto_name] = file_desc_proto + elif self._file_desc_protos_by_file[proto_name] != file_desc_proto: + raise DescriptorDatabaseConflictingDefinitionError( + '%s already added, but with different descriptor.' % proto_name) + else: + return + + # Add all the top-level descriptors to the index. + package = file_desc_proto.package + for message in file_desc_proto.message_type: + for name in _ExtractSymbols(message, package): + self._AddSymbol(name, file_desc_proto) + for enum in file_desc_proto.enum_type: + self._AddSymbol( + ('.'.join((package, enum.name)) if package else enum.name), + file_desc_proto, + ) + for enum_value in enum.value: + self._file_desc_protos_by_symbol[ + '.'.join((package, enum_value.name)) if package else enum_value.name + ] = file_desc_proto + for extension in file_desc_proto.extension: + self._AddSymbol( + ('.'.join((package, extension.name)) if package else extension.name), + file_desc_proto, + ) + for service in file_desc_proto.service: + self._AddSymbol( + ('.'.join((package, service.name)) if package else service.name), + file_desc_proto, + ) + + def FindFileByName(self, name): + """Finds the file descriptor proto by file name. + + Typically the file name is a relative path ending to a .proto file. The + proto with the given name will have to have been added to this database + using the Add method or else an error will be raised. + + Args: + name: The file name to find. + + Returns: + The file descriptor proto matching the name. + + Raises: + KeyError if no file by the given name was added. + """ + + return self._file_desc_protos_by_file[name] + + def FindFileContainingSymbol(self, symbol): + """Finds the file descriptor proto containing the specified symbol. + + The symbol should be a fully qualified name including the file descriptor's + package and any containing messages. Some examples: + + 'some.package.name.Message' + 'some.package.name.Message.NestedEnum' + 'some.package.name.Message.some_field' + + The file descriptor proto containing the specified symbol must be added to + this database using the Add method or else an error will be raised. + + Args: + symbol: The fully qualified symbol name. + + Returns: + The file descriptor proto containing the symbol. + + Raises: + KeyError if no file contains the specified symbol. + """ + if symbol.count('.') == 1 and symbol[0] == '.': + symbol = symbol.lstrip('.') + warnings.warn( + 'Please remove the leading "." when ' + 'FindFileContainingSymbol, this will turn to error ' + 'in 2026 Jan.', + RuntimeWarning, + ) + try: + return self._file_desc_protos_by_symbol[symbol] + except KeyError: + # Fields, enum values, and nested extensions are not in + # _file_desc_protos_by_symbol. Try to find the top level + # descriptor. Non-existent nested symbol under a valid top level + # descriptor can also be found. The behavior is the same with + # protobuf C++. + top_level, _, _ = symbol.rpartition('.') + try: + return self._file_desc_protos_by_symbol[top_level] + except KeyError: + # Raise the original symbol as a KeyError for better diagnostics. + raise KeyError(symbol) + + def FindFileContainingExtension(self, extendee_name, extension_number): + # TODO: implement this API. + return None + + def FindAllExtensionNumbers(self, extendee_name): + # TODO: implement this API. + return [] + + def _AddSymbol(self, name, file_desc_proto): + if name in self._file_desc_protos_by_symbol: + warn_msg = ('Conflict register for file "' + file_desc_proto.name + + '": ' + name + + ' is already defined in file "' + + self._file_desc_protos_by_symbol[name].name + '"') + warnings.warn(warn_msg, RuntimeWarning) + self._file_desc_protos_by_symbol[name] = file_desc_proto + + +def _ExtractSymbols(desc_proto, package): + """Pulls out all the symbols from a descriptor proto. + + Args: + desc_proto: The proto to extract symbols from. + package: The package containing the descriptor type. + + Yields: + The fully qualified name found in the descriptor. + """ + message_name = package + '.' + desc_proto.name if package else desc_proto.name + yield message_name + for nested_type in desc_proto.nested_type: + for symbol in _ExtractSymbols(nested_type, message_name): + yield symbol + for enum_type in desc_proto.enum_type: + yield '.'.join((message_name, enum_type.name)) diff --git a/lib/python3.12/site-packages/google/protobuf/descriptor_pb2.py b/lib/python3.12/site-packages/google/protobuf/descriptor_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..8aea28e2083ac37415ba72556e07155e60ae351f --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/descriptor_pb2.py @@ -0,0 +1,3369 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/descriptor.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/descriptor.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +if not _descriptor._USE_C_DESCRIPTORS: + DESCRIPTOR = _descriptor.FileDescriptor( + name='google/protobuf/descriptor.proto', + package='google.protobuf', + syntax='proto2', + edition='EDITION_PROTO2', + serialized_options=b'\n\023com.google.protobufB\020DescriptorProtosH\001Z-google.golang.org/protobuf/types/descriptorpb\370\001\001\242\002\003GPB\252\002\032Google.Protobuf.Reflection', + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n google/protobuf/descriptor.proto\x12\x0fgoogle.protobuf\"[\n\x11\x46ileDescriptorSet\x12\x38\n\x04\x66ile\x18\x01 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x04\x66ile*\x0c\x08\x80\xec\xca\xff\x01\x10\x81\xec\xca\xff\x01\"\xc5\x05\n\x13\x46ileDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n\x07package\x18\x02 \x01(\tR\x07package\x12\x1e\n\ndependency\x18\x03 \x03(\tR\ndependency\x12+\n\x11public_dependency\x18\n \x03(\x05R\x10publicDependency\x12\'\n\x0fweak_dependency\x18\x0b \x03(\x05R\x0eweakDependency\x12+\n\x11option_dependency\x18\x0f \x03(\tR\x10optionDependency\x12\x43\n\x0cmessage_type\x18\x04 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\x0bmessageType\x12\x41\n\tenum_type\x18\x05 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12\x41\n\x07service\x18\x06 \x03(\x0b\x32\'.google.protobuf.ServiceDescriptorProtoR\x07service\x12\x43\n\textension\x18\x07 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x36\n\x07options\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.FileOptionsR\x07options\x12I\n\x10source_code_info\x18\t \x01(\x0b\x32\x1f.google.protobuf.SourceCodeInfoR\x0esourceCodeInfo\x12\x16\n\x06syntax\x18\x0c \x01(\tR\x06syntax\x12\x32\n\x07\x65\x64ition\x18\x0e \x01(\x0e\x32\x18.google.protobuf.EditionR\x07\x65\x64ition\"\xfc\x06\n\x0f\x44\x65scriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12;\n\x05\x66ield\x18\x02 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\x05\x66ield\x12\x43\n\textension\x18\x06 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x41\n\x0bnested_type\x18\x03 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\nnestedType\x12\x41\n\tenum_type\x18\x04 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12X\n\x0f\x65xtension_range\x18\x05 \x03(\x0b\x32/.google.protobuf.DescriptorProto.ExtensionRangeR\x0e\x65xtensionRange\x12\x44\n\noneof_decl\x18\x08 \x03(\x0b\x32%.google.protobuf.OneofDescriptorProtoR\toneofDecl\x12\x39\n\x07options\x18\x07 \x01(\x0b\x32\x1f.google.protobuf.MessageOptionsR\x07options\x12U\n\x0ereserved_range\x18\t \x03(\x0b\x32..google.protobuf.DescriptorProto.ReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\n \x03(\tR\x0creservedName\x12\x41\n\nvisibility\x18\x0b \x01(\x0e\x32!.google.protobuf.SymbolVisibilityR\nvisibility\x1az\n\x0e\x45xtensionRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\x12@\n\x07options\x18\x03 \x01(\x0b\x32&.google.protobuf.ExtensionRangeOptionsR\x07options\x1a\x37\n\rReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\xcc\x04\n\x15\x45xtensionRangeOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\x12Y\n\x0b\x64\x65\x63laration\x18\x02 \x03(\x0b\x32\x32.google.protobuf.ExtensionRangeOptions.DeclarationB\x03\x88\x01\x02R\x0b\x64\x65\x63laration\x12\x37\n\x08\x66\x65\x61tures\x18\x32 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12m\n\x0cverification\x18\x03 \x01(\x0e\x32\x38.google.protobuf.ExtensionRangeOptions.VerificationState:\nUNVERIFIEDB\x03\x88\x01\x02R\x0cverification\x1a\x94\x01\n\x0b\x44\x65\x63laration\x12\x16\n\x06number\x18\x01 \x01(\x05R\x06number\x12\x1b\n\tfull_name\x18\x02 \x01(\tR\x08\x66ullName\x12\x12\n\x04type\x18\x03 \x01(\tR\x04type\x12\x1a\n\x08reserved\x18\x05 \x01(\x08R\x08reserved\x12\x1a\n\x08repeated\x18\x06 \x01(\x08R\x08repeatedJ\x04\x08\x04\x10\x05\"4\n\x11VerificationState\x12\x0f\n\x0b\x44\x45\x43LARATION\x10\x00\x12\x0e\n\nUNVERIFIED\x10\x01*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc1\x06\n\x14\x46ieldDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x41\n\x05label\x18\x04 \x01(\x0e\x32+.google.protobuf.FieldDescriptorProto.LabelR\x05label\x12>\n\x04type\x18\x05 \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.TypeR\x04type\x12\x1b\n\ttype_name\x18\x06 \x01(\tR\x08typeName\x12\x1a\n\x08\x65xtendee\x18\x02 \x01(\tR\x08\x65xtendee\x12#\n\rdefault_value\x18\x07 \x01(\tR\x0c\x64\x65\x66\x61ultValue\x12\x1f\n\x0boneof_index\x18\t \x01(\x05R\noneofIndex\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12\x37\n\x07options\x18\x08 \x01(\x0b\x32\x1d.google.protobuf.FieldOptionsR\x07options\x12\'\n\x0fproto3_optional\x18\x11 \x01(\x08R\x0eproto3Optional\"\xb6\x02\n\x04Type\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"C\n\x05Label\x12\x12\n\x0eLABEL_OPTIONAL\x10\x01\x12\x12\n\x0eLABEL_REPEATED\x10\x03\x12\x12\n\x0eLABEL_REQUIRED\x10\x02\"c\n\x14OneofDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\x07options\x18\x02 \x01(\x0b\x32\x1d.google.protobuf.OneofOptionsR\x07options\"\xa6\x03\n\x13\x45numDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12?\n\x05value\x18\x02 \x03(\x0b\x32).google.protobuf.EnumValueDescriptorProtoR\x05value\x12\x36\n\x07options\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.EnumOptionsR\x07options\x12]\n\x0ereserved_range\x18\x04 \x03(\x0b\x32\x36.google.protobuf.EnumDescriptorProto.EnumReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\x05 \x03(\tR\x0creservedName\x12\x41\n\nvisibility\x18\x06 \x01(\x0e\x32!.google.protobuf.SymbolVisibilityR\nvisibility\x1a;\n\x11\x45numReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\x83\x01\n\x18\x45numValueDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12;\n\x07options\x18\x03 \x01(\x0b\x32!.google.protobuf.EnumValueOptionsR\x07options\"\xb5\x01\n\x16ServiceDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12>\n\x06method\x18\x02 \x03(\x0b\x32&.google.protobuf.MethodDescriptorProtoR\x06method\x12\x39\n\x07options\x18\x03 \x01(\x0b\x32\x1f.google.protobuf.ServiceOptionsR\x07optionsJ\x04\x08\x04\x10\x05R\x06stream\"\x89\x02\n\x15MethodDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n\ninput_type\x18\x02 \x01(\tR\tinputType\x12\x1f\n\x0boutput_type\x18\x03 \x01(\tR\noutputType\x12\x38\n\x07options\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.MethodOptionsR\x07options\x12\x30\n\x10\x63lient_streaming\x18\x05 \x01(\x08:\x05\x66\x61lseR\x0f\x63lientStreaming\x12\x30\n\x10server_streaming\x18\x06 \x01(\x08:\x05\x66\x61lseR\x0fserverStreaming\"\xad\t\n\x0b\x46ileOptions\x12!\n\x0cjava_package\x18\x01 \x01(\tR\x0bjavaPackage\x12\x30\n\x14java_outer_classname\x18\x08 \x01(\tR\x12javaOuterClassname\x12\x35\n\x13java_multiple_files\x18\n \x01(\x08:\x05\x66\x61lseR\x11javaMultipleFiles\x12\x44\n\x1djava_generate_equals_and_hash\x18\x14 \x01(\x08\x42\x02\x18\x01R\x19javaGenerateEqualsAndHash\x12:\n\x16java_string_check_utf8\x18\x1b \x01(\x08:\x05\x66\x61lseR\x13javaStringCheckUtf8\x12S\n\x0coptimize_for\x18\t \x01(\x0e\x32).google.protobuf.FileOptions.OptimizeMode:\x05SPEEDR\x0boptimizeFor\x12\x1d\n\ngo_package\x18\x0b \x01(\tR\tgoPackage\x12\x35\n\x13\x63\x63_generic_services\x18\x10 \x01(\x08:\x05\x66\x61lseR\x11\x63\x63GenericServices\x12\x39\n\x15java_generic_services\x18\x11 \x01(\x08:\x05\x66\x61lseR\x13javaGenericServices\x12\x35\n\x13py_generic_services\x18\x12 \x01(\x08:\x05\x66\x61lseR\x11pyGenericServices\x12%\n\ndeprecated\x18\x17 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12.\n\x10\x63\x63_enable_arenas\x18\x1f \x01(\x08:\x04trueR\x0e\x63\x63\x45nableArenas\x12*\n\x11objc_class_prefix\x18$ \x01(\tR\x0fobjcClassPrefix\x12)\n\x10\x63sharp_namespace\x18% \x01(\tR\x0f\x63sharpNamespace\x12!\n\x0cswift_prefix\x18\' \x01(\tR\x0bswiftPrefix\x12(\n\x10php_class_prefix\x18( \x01(\tR\x0ephpClassPrefix\x12#\n\rphp_namespace\x18) \x01(\tR\x0cphpNamespace\x12\x34\n\x16php_metadata_namespace\x18, \x01(\tR\x14phpMetadataNamespace\x12!\n\x0cruby_package\x18- \x01(\tR\x0brubyPackage\x12\x37\n\x08\x66\x65\x61tures\x18\x32 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\":\n\x0cOptimizeMode\x12\t\n\x05SPEED\x10\x01\x12\r\n\tCODE_SIZE\x10\x02\x12\x10\n\x0cLITE_RUNTIME\x10\x03*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08*\x10+J\x04\x08&\x10\'R\x14php_generic_services\"\xf4\x03\n\x0eMessageOptions\x12<\n\x17message_set_wire_format\x18\x01 \x01(\x08:\x05\x66\x61lseR\x14messageSetWireFormat\x12L\n\x1fno_standard_descriptor_accessor\x18\x02 \x01(\x08:\x05\x66\x61lseR\x1cnoStandardDescriptorAccessor\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1b\n\tmap_entry\x18\x07 \x01(\x08R\x08mapEntry\x12V\n&deprecated_legacy_json_field_conflicts\x18\x0b \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12\x37\n\x08\x66\x65\x61tures\x18\x0c \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\tJ\x04\x08\t\x10\n\"\xa1\r\n\x0c\x46ieldOptions\x12\x41\n\x05\x63type\x18\x01 \x01(\x0e\x32#.google.protobuf.FieldOptions.CType:\x06STRINGR\x05\x63type\x12\x16\n\x06packed\x18\x02 \x01(\x08R\x06packed\x12G\n\x06jstype\x18\x06 \x01(\x0e\x32$.google.protobuf.FieldOptions.JSType:\tJS_NORMALR\x06jstype\x12\x19\n\x04lazy\x18\x05 \x01(\x08:\x05\x66\x61lseR\x04lazy\x12.\n\x0funverified_lazy\x18\x0f \x01(\x08:\x05\x66\x61lseR\x0eunverifiedLazy\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1d\n\x04weak\x18\n \x01(\x08:\x05\x66\x61lseB\x02\x18\x01R\x04weak\x12(\n\x0c\x64\x65\x62ug_redact\x18\x10 \x01(\x08:\x05\x66\x61lseR\x0b\x64\x65\x62ugRedact\x12K\n\tretention\x18\x11 \x01(\x0e\x32-.google.protobuf.FieldOptions.OptionRetentionR\tretention\x12H\n\x07targets\x18\x13 \x03(\x0e\x32..google.protobuf.FieldOptions.OptionTargetTypeR\x07targets\x12W\n\x10\x65\x64ition_defaults\x18\x14 \x03(\x0b\x32,.google.protobuf.FieldOptions.EditionDefaultR\x0f\x65\x64itionDefaults\x12\x37\n\x08\x66\x65\x61tures\x18\x15 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12U\n\x0f\x66\x65\x61ture_support\x18\x16 \x01(\x0b\x32,.google.protobuf.FieldOptions.FeatureSupportR\x0e\x66\x65\x61tureSupport\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\x1aZ\n\x0e\x45\x64itionDefault\x12\x32\n\x07\x65\x64ition\x18\x03 \x01(\x0e\x32\x18.google.protobuf.EditionR\x07\x65\x64ition\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\x96\x02\n\x0e\x46\x65\x61tureSupport\x12G\n\x12\x65\x64ition_introduced\x18\x01 \x01(\x0e\x32\x18.google.protobuf.EditionR\x11\x65\x64itionIntroduced\x12G\n\x12\x65\x64ition_deprecated\x18\x02 \x01(\x0e\x32\x18.google.protobuf.EditionR\x11\x65\x64itionDeprecated\x12/\n\x13\x64\x65precation_warning\x18\x03 \x01(\tR\x12\x64\x65precationWarning\x12\x41\n\x0f\x65\x64ition_removed\x18\x04 \x01(\x0e\x32\x18.google.protobuf.EditionR\x0e\x65\x64itionRemoved\"/\n\x05\x43Type\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x43ORD\x10\x01\x12\x10\n\x0cSTRING_PIECE\x10\x02\"5\n\x06JSType\x12\r\n\tJS_NORMAL\x10\x00\x12\r\n\tJS_STRING\x10\x01\x12\r\n\tJS_NUMBER\x10\x02\"U\n\x0fOptionRetention\x12\x15\n\x11RETENTION_UNKNOWN\x10\x00\x12\x15\n\x11RETENTION_RUNTIME\x10\x01\x12\x14\n\x10RETENTION_SOURCE\x10\x02\"\x8c\x02\n\x10OptionTargetType\x12\x17\n\x13TARGET_TYPE_UNKNOWN\x10\x00\x12\x14\n\x10TARGET_TYPE_FILE\x10\x01\x12\x1f\n\x1bTARGET_TYPE_EXTENSION_RANGE\x10\x02\x12\x17\n\x13TARGET_TYPE_MESSAGE\x10\x03\x12\x15\n\x11TARGET_TYPE_FIELD\x10\x04\x12\x15\n\x11TARGET_TYPE_ONEOF\x10\x05\x12\x14\n\x10TARGET_TYPE_ENUM\x10\x06\x12\x1a\n\x16TARGET_TYPE_ENUM_ENTRY\x10\x07\x12\x17\n\x13TARGET_TYPE_SERVICE\x10\x08\x12\x16\n\x12TARGET_TYPE_METHOD\x10\t*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13\"\xac\x01\n\x0cOneofOptions\x12\x37\n\x08\x66\x65\x61tures\x18\x01 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xd1\x02\n\x0b\x45numOptions\x12\x1f\n\x0b\x61llow_alias\x18\x02 \x01(\x08R\nallowAlias\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12V\n&deprecated_legacy_json_field_conflicts\x18\x06 \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12\x37\n\x08\x66\x65\x61tures\x18\x07 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x05\x10\x06\"\xd8\x02\n\x10\x45numValueOptions\x12%\n\ndeprecated\x18\x01 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x37\n\x08\x66\x65\x61tures\x18\x02 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12(\n\x0c\x64\x65\x62ug_redact\x18\x03 \x01(\x08:\x05\x66\x61lseR\x0b\x64\x65\x62ugRedact\x12U\n\x0f\x66\x65\x61ture_support\x18\x04 \x01(\x0b\x32,.google.protobuf.FieldOptions.FeatureSupportR\x0e\x66\x65\x61tureSupport\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xd5\x01\n\x0eServiceOptions\x12\x37\n\x08\x66\x65\x61tures\x18\" \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x99\x03\n\rMethodOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12q\n\x11idempotency_level\x18\" \x01(\x0e\x32/.google.protobuf.MethodOptions.IdempotencyLevel:\x13IDEMPOTENCY_UNKNOWNR\x10idempotencyLevel\x12\x37\n\x08\x66\x65\x61tures\x18# \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"P\n\x10IdempotencyLevel\x12\x17\n\x13IDEMPOTENCY_UNKNOWN\x10\x00\x12\x13\n\x0fNO_SIDE_EFFECTS\x10\x01\x12\x0e\n\nIDEMPOTENT\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9a\x03\n\x13UninterpretedOption\x12\x41\n\x04name\x18\x02 \x03(\x0b\x32-.google.protobuf.UninterpretedOption.NamePartR\x04name\x12)\n\x10identifier_value\x18\x03 \x01(\tR\x0fidentifierValue\x12,\n\x12positive_int_value\x18\x04 \x01(\x04R\x10positiveIntValue\x12,\n\x12negative_int_value\x18\x05 \x01(\x03R\x10negativeIntValue\x12!\n\x0c\x64ouble_value\x18\x06 \x01(\x01R\x0b\x64oubleValue\x12!\n\x0cstring_value\x18\x07 \x01(\x0cR\x0bstringValue\x12\'\n\x0f\x61ggregate_value\x18\x08 \x01(\tR\x0e\x61ggregateValue\x1aJ\n\x08NamePart\x12\x1b\n\tname_part\x18\x01 \x02(\tR\x08namePart\x12!\n\x0cis_extension\x18\x02 \x02(\x08R\x0bisExtension\"\x8e\x0f\n\nFeatureSet\x12\x91\x01\n\x0e\x66ield_presence\x18\x01 \x01(\x0e\x32).google.protobuf.FeatureSet.FieldPresenceB?\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\r\x12\x08\x45XPLICIT\x18\x84\x07\xa2\x01\r\x12\x08IMPLICIT\x18\xe7\x07\xa2\x01\r\x12\x08\x45XPLICIT\x18\xe8\x07\xb2\x01\x03\x08\xe8\x07R\rfieldPresence\x12l\n\tenum_type\x18\x02 \x01(\x0e\x32$.google.protobuf.FeatureSet.EnumTypeB)\x88\x01\x01\x98\x01\x06\x98\x01\x01\xa2\x01\x0b\x12\x06\x43LOSED\x18\x84\x07\xa2\x01\t\x12\x04OPEN\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\x08\x65numType\x12\x98\x01\n\x17repeated_field_encoding\x18\x03 \x01(\x0e\x32\x31.google.protobuf.FeatureSet.RepeatedFieldEncodingB-\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\r\x12\x08\x45XPANDED\x18\x84\x07\xa2\x01\x0b\x12\x06PACKED\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\x15repeatedFieldEncoding\x12~\n\x0futf8_validation\x18\x04 \x01(\x0e\x32*.google.protobuf.FeatureSet.Utf8ValidationB)\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\t\x12\x04NONE\x18\x84\x07\xa2\x01\x0b\x12\x06VERIFY\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\x0eutf8Validation\x12~\n\x10message_encoding\x18\x05 \x01(\x0e\x32+.google.protobuf.FeatureSet.MessageEncodingB&\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\x14\x12\x0fLENGTH_PREFIXED\x18\x84\x07\xb2\x01\x03\x08\xe8\x07R\x0fmessageEncoding\x12\x82\x01\n\x0bjson_format\x18\x06 \x01(\x0e\x32&.google.protobuf.FeatureSet.JsonFormatB9\x88\x01\x01\x98\x01\x03\x98\x01\x06\x98\x01\x01\xa2\x01\x17\x12\x12LEGACY_BEST_EFFORT\x18\x84\x07\xa2\x01\n\x12\x05\x41LLOW\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\njsonFormat\x12\xab\x01\n\x14\x65nforce_naming_style\x18\x07 \x01(\x0e\x32..google.protobuf.FeatureSet.EnforceNamingStyleBI\x88\x01\x02\x98\x01\x01\x98\x01\x02\x98\x01\x03\x98\x01\x04\x98\x01\x05\x98\x01\x06\x98\x01\x07\x98\x01\x08\x98\x01\t\xa2\x01\x11\x12\x0cSTYLE_LEGACY\x18\x84\x07\xa2\x01\x0e\x12\tSTYLE2024\x18\xe9\x07\xb2\x01\x03\x08\xe9\x07R\x12\x65nforceNamingStyle\x12\xb9\x01\n\x19\x64\x65\x66\x61ult_symbol_visibility\x18\x08 \x01(\x0e\x32\x45.google.protobuf.FeatureSet.VisibilityFeature.DefaultSymbolVisibilityB6\x88\x01\x02\x98\x01\x01\xa2\x01\x0f\x12\nEXPORT_ALL\x18\x84\x07\xa2\x01\x15\x12\x10\x45XPORT_TOP_LEVEL\x18\xe9\x07\xb2\x01\x03\x08\xe9\x07R\x17\x64\x65\x66\x61ultSymbolVisibility\x1a\xa1\x01\n\x11VisibilityFeature\"\x81\x01\n\x17\x44\x65\x66\x61ultSymbolVisibility\x12%\n!DEFAULT_SYMBOL_VISIBILITY_UNKNOWN\x10\x00\x12\x0e\n\nEXPORT_ALL\x10\x01\x12\x14\n\x10\x45XPORT_TOP_LEVEL\x10\x02\x12\r\n\tLOCAL_ALL\x10\x03\x12\n\n\x06STRICT\x10\x04J\x08\x08\x01\x10\x80\x80\x80\x80\x02\"\\\n\rFieldPresence\x12\x1a\n\x16\x46IELD_PRESENCE_UNKNOWN\x10\x00\x12\x0c\n\x08\x45XPLICIT\x10\x01\x12\x0c\n\x08IMPLICIT\x10\x02\x12\x13\n\x0fLEGACY_REQUIRED\x10\x03\"7\n\x08\x45numType\x12\x15\n\x11\x45NUM_TYPE_UNKNOWN\x10\x00\x12\x08\n\x04OPEN\x10\x01\x12\n\n\x06\x43LOSED\x10\x02\"V\n\x15RepeatedFieldEncoding\x12#\n\x1fREPEATED_FIELD_ENCODING_UNKNOWN\x10\x00\x12\n\n\x06PACKED\x10\x01\x12\x0c\n\x08\x45XPANDED\x10\x02\"I\n\x0eUtf8Validation\x12\x1b\n\x17UTF8_VALIDATION_UNKNOWN\x10\x00\x12\n\n\x06VERIFY\x10\x02\x12\x08\n\x04NONE\x10\x03\"\x04\x08\x01\x10\x01\"S\n\x0fMessageEncoding\x12\x1c\n\x18MESSAGE_ENCODING_UNKNOWN\x10\x00\x12\x13\n\x0fLENGTH_PREFIXED\x10\x01\x12\r\n\tDELIMITED\x10\x02\"H\n\nJsonFormat\x12\x17\n\x13JSON_FORMAT_UNKNOWN\x10\x00\x12\t\n\x05\x41LLOW\x10\x01\x12\x16\n\x12LEGACY_BEST_EFFORT\x10\x02\"W\n\x12\x45nforceNamingStyle\x12 \n\x1c\x45NFORCE_NAMING_STYLE_UNKNOWN\x10\x00\x12\r\n\tSTYLE2024\x10\x01\x12\x10\n\x0cSTYLE_LEGACY\x10\x02*\x06\x08\xe8\x07\x10\x8bN*\x06\x08\x8bN\x10\x90N*\x06\x08\x90N\x10\x91NJ\x06\x08\xe7\x07\x10\xe8\x07\"\xef\x03\n\x12\x46\x65\x61tureSetDefaults\x12X\n\x08\x64\x65\x66\x61ults\x18\x01 \x03(\x0b\x32<.google.protobuf.FeatureSetDefaults.FeatureSetEditionDefaultR\x08\x64\x65\x66\x61ults\x12\x41\n\x0fminimum_edition\x18\x04 \x01(\x0e\x32\x18.google.protobuf.EditionR\x0eminimumEdition\x12\x41\n\x0fmaximum_edition\x18\x05 \x01(\x0e\x32\x18.google.protobuf.EditionR\x0emaximumEdition\x1a\xf8\x01\n\x18\x46\x65\x61tureSetEditionDefault\x12\x32\n\x07\x65\x64ition\x18\x03 \x01(\x0e\x32\x18.google.protobuf.EditionR\x07\x65\x64ition\x12N\n\x14overridable_features\x18\x04 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x13overridableFeatures\x12\x42\n\x0e\x66ixed_features\x18\x05 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\rfixedFeaturesJ\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03R\x08\x66\x65\x61tures\"\xb5\x02\n\x0eSourceCodeInfo\x12\x44\n\x08location\x18\x01 \x03(\x0b\x32(.google.protobuf.SourceCodeInfo.LocationR\x08location\x1a\xce\x01\n\x08Location\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x16\n\x04span\x18\x02 \x03(\x05\x42\x02\x10\x01R\x04span\x12)\n\x10leading_comments\x18\x03 \x01(\tR\x0fleadingComments\x12+\n\x11trailing_comments\x18\x04 \x01(\tR\x10trailingComments\x12:\n\x19leading_detached_comments\x18\x06 \x03(\tR\x17leadingDetachedComments*\x0c\x08\x80\xec\xca\xff\x01\x10\x81\xec\xca\xff\x01\"\xd0\x02\n\x11GeneratedCodeInfo\x12M\n\nannotation\x18\x01 \x03(\x0b\x32-.google.protobuf.GeneratedCodeInfo.AnnotationR\nannotation\x1a\xeb\x01\n\nAnnotation\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x1f\n\x0bsource_file\x18\x02 \x01(\tR\nsourceFile\x12\x14\n\x05\x62\x65gin\x18\x03 \x01(\x05R\x05\x62\x65gin\x12\x10\n\x03\x65nd\x18\x04 \x01(\x05R\x03\x65nd\x12R\n\x08semantic\x18\x05 \x01(\x0e\x32\x36.google.protobuf.GeneratedCodeInfo.Annotation.SemanticR\x08semantic\"(\n\x08Semantic\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03SET\x10\x01\x12\t\n\x05\x41LIAS\x10\x02*\xbe\x02\n\x07\x45\x64ition\x12\x13\n\x0f\x45\x44ITION_UNKNOWN\x10\x00\x12\x13\n\x0e\x45\x44ITION_LEGACY\x10\x84\x07\x12\x13\n\x0e\x45\x44ITION_PROTO2\x10\xe6\x07\x12\x13\n\x0e\x45\x44ITION_PROTO3\x10\xe7\x07\x12\x11\n\x0c\x45\x44ITION_2023\x10\xe8\x07\x12\x11\n\x0c\x45\x44ITION_2024\x10\xe9\x07\x12\x15\n\x10\x45\x44ITION_UNSTABLE\x10\x8fN\x12\x17\n\x13\x45\x44ITION_1_TEST_ONLY\x10\x01\x12\x17\n\x13\x45\x44ITION_2_TEST_ONLY\x10\x02\x12\x1d\n\x17\x45\x44ITION_99997_TEST_ONLY\x10\x9d\x8d\x06\x12\x1d\n\x17\x45\x44ITION_99998_TEST_ONLY\x10\x9e\x8d\x06\x12\x1d\n\x17\x45\x44ITION_99999_TEST_ONLY\x10\x9f\x8d\x06\x12\x13\n\x0b\x45\x44ITION_MAX\x10\xff\xff\xff\xff\x07*U\n\x10SymbolVisibility\x12\x14\n\x10VISIBILITY_UNSET\x10\x00\x12\x14\n\x10VISIBILITY_LOCAL\x10\x01\x12\x15\n\x11VISIBILITY_EXPORT\x10\x02\x42~\n\x13\x63om.google.protobufB\x10\x44\x65scriptorProtosH\x01Z-google.golang.org/protobuf/types/descriptorpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1aGoogle.Protobuf.Reflection' + ) +else: + DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n google/protobuf/descriptor.proto\x12\x0fgoogle.protobuf\"[\n\x11\x46ileDescriptorSet\x12\x38\n\x04\x66ile\x18\x01 \x03(\x0b\x32$.google.protobuf.FileDescriptorProtoR\x04\x66ile*\x0c\x08\x80\xec\xca\xff\x01\x10\x81\xec\xca\xff\x01\"\xc5\x05\n\x13\x46ileDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x18\n\x07package\x18\x02 \x01(\tR\x07package\x12\x1e\n\ndependency\x18\x03 \x03(\tR\ndependency\x12+\n\x11public_dependency\x18\n \x03(\x05R\x10publicDependency\x12\'\n\x0fweak_dependency\x18\x0b \x03(\x05R\x0eweakDependency\x12+\n\x11option_dependency\x18\x0f \x03(\tR\x10optionDependency\x12\x43\n\x0cmessage_type\x18\x04 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\x0bmessageType\x12\x41\n\tenum_type\x18\x05 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12\x41\n\x07service\x18\x06 \x03(\x0b\x32\'.google.protobuf.ServiceDescriptorProtoR\x07service\x12\x43\n\textension\x18\x07 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x36\n\x07options\x18\x08 \x01(\x0b\x32\x1c.google.protobuf.FileOptionsR\x07options\x12I\n\x10source_code_info\x18\t \x01(\x0b\x32\x1f.google.protobuf.SourceCodeInfoR\x0esourceCodeInfo\x12\x16\n\x06syntax\x18\x0c \x01(\tR\x06syntax\x12\x32\n\x07\x65\x64ition\x18\x0e \x01(\x0e\x32\x18.google.protobuf.EditionR\x07\x65\x64ition\"\xfc\x06\n\x0f\x44\x65scriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12;\n\x05\x66ield\x18\x02 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\x05\x66ield\x12\x43\n\textension\x18\x06 \x03(\x0b\x32%.google.protobuf.FieldDescriptorProtoR\textension\x12\x41\n\x0bnested_type\x18\x03 \x03(\x0b\x32 .google.protobuf.DescriptorProtoR\nnestedType\x12\x41\n\tenum_type\x18\x04 \x03(\x0b\x32$.google.protobuf.EnumDescriptorProtoR\x08\x65numType\x12X\n\x0f\x65xtension_range\x18\x05 \x03(\x0b\x32/.google.protobuf.DescriptorProto.ExtensionRangeR\x0e\x65xtensionRange\x12\x44\n\noneof_decl\x18\x08 \x03(\x0b\x32%.google.protobuf.OneofDescriptorProtoR\toneofDecl\x12\x39\n\x07options\x18\x07 \x01(\x0b\x32\x1f.google.protobuf.MessageOptionsR\x07options\x12U\n\x0ereserved_range\x18\t \x03(\x0b\x32..google.protobuf.DescriptorProto.ReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\n \x03(\tR\x0creservedName\x12\x41\n\nvisibility\x18\x0b \x01(\x0e\x32!.google.protobuf.SymbolVisibilityR\nvisibility\x1az\n\x0e\x45xtensionRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\x12@\n\x07options\x18\x03 \x01(\x0b\x32&.google.protobuf.ExtensionRangeOptionsR\x07options\x1a\x37\n\rReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\xcc\x04\n\x15\x45xtensionRangeOptions\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\x12Y\n\x0b\x64\x65\x63laration\x18\x02 \x03(\x0b\x32\x32.google.protobuf.ExtensionRangeOptions.DeclarationB\x03\x88\x01\x02R\x0b\x64\x65\x63laration\x12\x37\n\x08\x66\x65\x61tures\x18\x32 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12m\n\x0cverification\x18\x03 \x01(\x0e\x32\x38.google.protobuf.ExtensionRangeOptions.VerificationState:\nUNVERIFIEDB\x03\x88\x01\x02R\x0cverification\x1a\x94\x01\n\x0b\x44\x65\x63laration\x12\x16\n\x06number\x18\x01 \x01(\x05R\x06number\x12\x1b\n\tfull_name\x18\x02 \x01(\tR\x08\x66ullName\x12\x12\n\x04type\x18\x03 \x01(\tR\x04type\x12\x1a\n\x08reserved\x18\x05 \x01(\x08R\x08reserved\x12\x1a\n\x08repeated\x18\x06 \x01(\x08R\x08repeatedJ\x04\x08\x04\x10\x05\"4\n\x11VerificationState\x12\x0f\n\x0b\x44\x45\x43LARATION\x10\x00\x12\x0e\n\nUNVERIFIED\x10\x01*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xc1\x06\n\x14\x46ieldDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x41\n\x05label\x18\x04 \x01(\x0e\x32+.google.protobuf.FieldDescriptorProto.LabelR\x05label\x12>\n\x04type\x18\x05 \x01(\x0e\x32*.google.protobuf.FieldDescriptorProto.TypeR\x04type\x12\x1b\n\ttype_name\x18\x06 \x01(\tR\x08typeName\x12\x1a\n\x08\x65xtendee\x18\x02 \x01(\tR\x08\x65xtendee\x12#\n\rdefault_value\x18\x07 \x01(\tR\x0c\x64\x65\x66\x61ultValue\x12\x1f\n\x0boneof_index\x18\t \x01(\x05R\noneofIndex\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12\x37\n\x07options\x18\x08 \x01(\x0b\x32\x1d.google.protobuf.FieldOptionsR\x07options\x12\'\n\x0fproto3_optional\x18\x11 \x01(\x08R\x0eproto3Optional\"\xb6\x02\n\x04Type\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"C\n\x05Label\x12\x12\n\x0eLABEL_OPTIONAL\x10\x01\x12\x12\n\x0eLABEL_REPEATED\x10\x03\x12\x12\n\x0eLABEL_REQUIRED\x10\x02\"c\n\x14OneofDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x37\n\x07options\x18\x02 \x01(\x0b\x32\x1d.google.protobuf.OneofOptionsR\x07options\"\xa6\x03\n\x13\x45numDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12?\n\x05value\x18\x02 \x03(\x0b\x32).google.protobuf.EnumValueDescriptorProtoR\x05value\x12\x36\n\x07options\x18\x03 \x01(\x0b\x32\x1c.google.protobuf.EnumOptionsR\x07options\x12]\n\x0ereserved_range\x18\x04 \x03(\x0b\x32\x36.google.protobuf.EnumDescriptorProto.EnumReservedRangeR\rreservedRange\x12#\n\rreserved_name\x18\x05 \x03(\tR\x0creservedName\x12\x41\n\nvisibility\x18\x06 \x01(\x0e\x32!.google.protobuf.SymbolVisibilityR\nvisibility\x1a;\n\x11\x45numReservedRange\x12\x14\n\x05start\x18\x01 \x01(\x05R\x05start\x12\x10\n\x03\x65nd\x18\x02 \x01(\x05R\x03\x65nd\"\x83\x01\n\x18\x45numValueDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12;\n\x07options\x18\x03 \x01(\x0b\x32!.google.protobuf.EnumValueOptionsR\x07options\"\xb5\x01\n\x16ServiceDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12>\n\x06method\x18\x02 \x03(\x0b\x32&.google.protobuf.MethodDescriptorProtoR\x06method\x12\x39\n\x07options\x18\x03 \x01(\x0b\x32\x1f.google.protobuf.ServiceOptionsR\x07optionsJ\x04\x08\x04\x10\x05R\x06stream\"\x89\x02\n\x15MethodDescriptorProto\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n\ninput_type\x18\x02 \x01(\tR\tinputType\x12\x1f\n\x0boutput_type\x18\x03 \x01(\tR\noutputType\x12\x38\n\x07options\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.MethodOptionsR\x07options\x12\x30\n\x10\x63lient_streaming\x18\x05 \x01(\x08:\x05\x66\x61lseR\x0f\x63lientStreaming\x12\x30\n\x10server_streaming\x18\x06 \x01(\x08:\x05\x66\x61lseR\x0fserverStreaming\"\xad\t\n\x0b\x46ileOptions\x12!\n\x0cjava_package\x18\x01 \x01(\tR\x0bjavaPackage\x12\x30\n\x14java_outer_classname\x18\x08 \x01(\tR\x12javaOuterClassname\x12\x35\n\x13java_multiple_files\x18\n \x01(\x08:\x05\x66\x61lseR\x11javaMultipleFiles\x12\x44\n\x1djava_generate_equals_and_hash\x18\x14 \x01(\x08\x42\x02\x18\x01R\x19javaGenerateEqualsAndHash\x12:\n\x16java_string_check_utf8\x18\x1b \x01(\x08:\x05\x66\x61lseR\x13javaStringCheckUtf8\x12S\n\x0coptimize_for\x18\t \x01(\x0e\x32).google.protobuf.FileOptions.OptimizeMode:\x05SPEEDR\x0boptimizeFor\x12\x1d\n\ngo_package\x18\x0b \x01(\tR\tgoPackage\x12\x35\n\x13\x63\x63_generic_services\x18\x10 \x01(\x08:\x05\x66\x61lseR\x11\x63\x63GenericServices\x12\x39\n\x15java_generic_services\x18\x11 \x01(\x08:\x05\x66\x61lseR\x13javaGenericServices\x12\x35\n\x13py_generic_services\x18\x12 \x01(\x08:\x05\x66\x61lseR\x11pyGenericServices\x12%\n\ndeprecated\x18\x17 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12.\n\x10\x63\x63_enable_arenas\x18\x1f \x01(\x08:\x04trueR\x0e\x63\x63\x45nableArenas\x12*\n\x11objc_class_prefix\x18$ \x01(\tR\x0fobjcClassPrefix\x12)\n\x10\x63sharp_namespace\x18% \x01(\tR\x0f\x63sharpNamespace\x12!\n\x0cswift_prefix\x18\' \x01(\tR\x0bswiftPrefix\x12(\n\x10php_class_prefix\x18( \x01(\tR\x0ephpClassPrefix\x12#\n\rphp_namespace\x18) \x01(\tR\x0cphpNamespace\x12\x34\n\x16php_metadata_namespace\x18, \x01(\tR\x14phpMetadataNamespace\x12!\n\x0cruby_package\x18- \x01(\tR\x0brubyPackage\x12\x37\n\x08\x66\x65\x61tures\x18\x32 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\":\n\x0cOptimizeMode\x12\t\n\x05SPEED\x10\x01\x12\r\n\tCODE_SIZE\x10\x02\x12\x10\n\x0cLITE_RUNTIME\x10\x03*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08*\x10+J\x04\x08&\x10\'R\x14php_generic_services\"\xf4\x03\n\x0eMessageOptions\x12<\n\x17message_set_wire_format\x18\x01 \x01(\x08:\x05\x66\x61lseR\x14messageSetWireFormat\x12L\n\x1fno_standard_descriptor_accessor\x18\x02 \x01(\x08:\x05\x66\x61lseR\x1cnoStandardDescriptorAccessor\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1b\n\tmap_entry\x18\x07 \x01(\x08R\x08mapEntry\x12V\n&deprecated_legacy_json_field_conflicts\x18\x0b \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12\x37\n\x08\x66\x65\x61tures\x18\x0c \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x05\x10\x06J\x04\x08\x06\x10\x07J\x04\x08\x08\x10\tJ\x04\x08\t\x10\n\"\xa1\r\n\x0c\x46ieldOptions\x12\x41\n\x05\x63type\x18\x01 \x01(\x0e\x32#.google.protobuf.FieldOptions.CType:\x06STRINGR\x05\x63type\x12\x16\n\x06packed\x18\x02 \x01(\x08R\x06packed\x12G\n\x06jstype\x18\x06 \x01(\x0e\x32$.google.protobuf.FieldOptions.JSType:\tJS_NORMALR\x06jstype\x12\x19\n\x04lazy\x18\x05 \x01(\x08:\x05\x66\x61lseR\x04lazy\x12.\n\x0funverified_lazy\x18\x0f \x01(\x08:\x05\x66\x61lseR\x0eunverifiedLazy\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x1d\n\x04weak\x18\n \x01(\x08:\x05\x66\x61lseB\x02\x18\x01R\x04weak\x12(\n\x0c\x64\x65\x62ug_redact\x18\x10 \x01(\x08:\x05\x66\x61lseR\x0b\x64\x65\x62ugRedact\x12K\n\tretention\x18\x11 \x01(\x0e\x32-.google.protobuf.FieldOptions.OptionRetentionR\tretention\x12H\n\x07targets\x18\x13 \x03(\x0e\x32..google.protobuf.FieldOptions.OptionTargetTypeR\x07targets\x12W\n\x10\x65\x64ition_defaults\x18\x14 \x03(\x0b\x32,.google.protobuf.FieldOptions.EditionDefaultR\x0f\x65\x64itionDefaults\x12\x37\n\x08\x66\x65\x61tures\x18\x15 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12U\n\x0f\x66\x65\x61ture_support\x18\x16 \x01(\x0b\x32,.google.protobuf.FieldOptions.FeatureSupportR\x0e\x66\x65\x61tureSupport\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\x1aZ\n\x0e\x45\x64itionDefault\x12\x32\n\x07\x65\x64ition\x18\x03 \x01(\x0e\x32\x18.google.protobuf.EditionR\x07\x65\x64ition\x12\x14\n\x05value\x18\x02 \x01(\tR\x05value\x1a\x96\x02\n\x0e\x46\x65\x61tureSupport\x12G\n\x12\x65\x64ition_introduced\x18\x01 \x01(\x0e\x32\x18.google.protobuf.EditionR\x11\x65\x64itionIntroduced\x12G\n\x12\x65\x64ition_deprecated\x18\x02 \x01(\x0e\x32\x18.google.protobuf.EditionR\x11\x65\x64itionDeprecated\x12/\n\x13\x64\x65precation_warning\x18\x03 \x01(\tR\x12\x64\x65precationWarning\x12\x41\n\x0f\x65\x64ition_removed\x18\x04 \x01(\x0e\x32\x18.google.protobuf.EditionR\x0e\x65\x64itionRemoved\"/\n\x05\x43Type\x12\n\n\x06STRING\x10\x00\x12\x08\n\x04\x43ORD\x10\x01\x12\x10\n\x0cSTRING_PIECE\x10\x02\"5\n\x06JSType\x12\r\n\tJS_NORMAL\x10\x00\x12\r\n\tJS_STRING\x10\x01\x12\r\n\tJS_NUMBER\x10\x02\"U\n\x0fOptionRetention\x12\x15\n\x11RETENTION_UNKNOWN\x10\x00\x12\x15\n\x11RETENTION_RUNTIME\x10\x01\x12\x14\n\x10RETENTION_SOURCE\x10\x02\"\x8c\x02\n\x10OptionTargetType\x12\x17\n\x13TARGET_TYPE_UNKNOWN\x10\x00\x12\x14\n\x10TARGET_TYPE_FILE\x10\x01\x12\x1f\n\x1bTARGET_TYPE_EXTENSION_RANGE\x10\x02\x12\x17\n\x13TARGET_TYPE_MESSAGE\x10\x03\x12\x15\n\x11TARGET_TYPE_FIELD\x10\x04\x12\x15\n\x11TARGET_TYPE_ONEOF\x10\x05\x12\x14\n\x10TARGET_TYPE_ENUM\x10\x06\x12\x1a\n\x16TARGET_TYPE_ENUM_ENTRY\x10\x07\x12\x17\n\x13TARGET_TYPE_SERVICE\x10\x08\x12\x16\n\x12TARGET_TYPE_METHOD\x10\t*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x04\x10\x05J\x04\x08\x12\x10\x13\"\xac\x01\n\x0cOneofOptions\x12\x37\n\x08\x66\x65\x61tures\x18\x01 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xd1\x02\n\x0b\x45numOptions\x12\x1f\n\x0b\x61llow_alias\x18\x02 \x01(\x08R\nallowAlias\x12%\n\ndeprecated\x18\x03 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12V\n&deprecated_legacy_json_field_conflicts\x18\x06 \x01(\x08\x42\x02\x18\x01R\"deprecatedLegacyJsonFieldConflicts\x12\x37\n\x08\x66\x65\x61tures\x18\x07 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02J\x04\x08\x05\x10\x06\"\xd8\x02\n\x10\x45numValueOptions\x12%\n\ndeprecated\x18\x01 \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12\x37\n\x08\x66\x65\x61tures\x18\x02 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12(\n\x0c\x64\x65\x62ug_redact\x18\x03 \x01(\x08:\x05\x66\x61lseR\x0b\x64\x65\x62ugRedact\x12U\n\x0f\x66\x65\x61ture_support\x18\x04 \x01(\x0b\x32,.google.protobuf.FieldOptions.FeatureSupportR\x0e\x66\x65\x61tureSupport\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\xd5\x01\n\x0eServiceOptions\x12\x37\n\x08\x66\x65\x61tures\x18\" \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x99\x03\n\rMethodOptions\x12%\n\ndeprecated\x18! \x01(\x08:\x05\x66\x61lseR\ndeprecated\x12q\n\x11idempotency_level\x18\" \x01(\x0e\x32/.google.protobuf.MethodOptions.IdempotencyLevel:\x13IDEMPOTENCY_UNKNOWNR\x10idempotencyLevel\x12\x37\n\x08\x66\x65\x61tures\x18# \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x08\x66\x65\x61tures\x12X\n\x14uninterpreted_option\x18\xe7\x07 \x03(\x0b\x32$.google.protobuf.UninterpretedOptionR\x13uninterpretedOption\"P\n\x10IdempotencyLevel\x12\x17\n\x13IDEMPOTENCY_UNKNOWN\x10\x00\x12\x13\n\x0fNO_SIDE_EFFECTS\x10\x01\x12\x0e\n\nIDEMPOTENT\x10\x02*\t\x08\xe8\x07\x10\x80\x80\x80\x80\x02\"\x9a\x03\n\x13UninterpretedOption\x12\x41\n\x04name\x18\x02 \x03(\x0b\x32-.google.protobuf.UninterpretedOption.NamePartR\x04name\x12)\n\x10identifier_value\x18\x03 \x01(\tR\x0fidentifierValue\x12,\n\x12positive_int_value\x18\x04 \x01(\x04R\x10positiveIntValue\x12,\n\x12negative_int_value\x18\x05 \x01(\x03R\x10negativeIntValue\x12!\n\x0c\x64ouble_value\x18\x06 \x01(\x01R\x0b\x64oubleValue\x12!\n\x0cstring_value\x18\x07 \x01(\x0cR\x0bstringValue\x12\'\n\x0f\x61ggregate_value\x18\x08 \x01(\tR\x0e\x61ggregateValue\x1aJ\n\x08NamePart\x12\x1b\n\tname_part\x18\x01 \x02(\tR\x08namePart\x12!\n\x0cis_extension\x18\x02 \x02(\x08R\x0bisExtension\"\x8e\x0f\n\nFeatureSet\x12\x91\x01\n\x0e\x66ield_presence\x18\x01 \x01(\x0e\x32).google.protobuf.FeatureSet.FieldPresenceB?\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\r\x12\x08\x45XPLICIT\x18\x84\x07\xa2\x01\r\x12\x08IMPLICIT\x18\xe7\x07\xa2\x01\r\x12\x08\x45XPLICIT\x18\xe8\x07\xb2\x01\x03\x08\xe8\x07R\rfieldPresence\x12l\n\tenum_type\x18\x02 \x01(\x0e\x32$.google.protobuf.FeatureSet.EnumTypeB)\x88\x01\x01\x98\x01\x06\x98\x01\x01\xa2\x01\x0b\x12\x06\x43LOSED\x18\x84\x07\xa2\x01\t\x12\x04OPEN\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\x08\x65numType\x12\x98\x01\n\x17repeated_field_encoding\x18\x03 \x01(\x0e\x32\x31.google.protobuf.FeatureSet.RepeatedFieldEncodingB-\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\r\x12\x08\x45XPANDED\x18\x84\x07\xa2\x01\x0b\x12\x06PACKED\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\x15repeatedFieldEncoding\x12~\n\x0futf8_validation\x18\x04 \x01(\x0e\x32*.google.protobuf.FeatureSet.Utf8ValidationB)\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\t\x12\x04NONE\x18\x84\x07\xa2\x01\x0b\x12\x06VERIFY\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\x0eutf8Validation\x12~\n\x10message_encoding\x18\x05 \x01(\x0e\x32+.google.protobuf.FeatureSet.MessageEncodingB&\x88\x01\x01\x98\x01\x04\x98\x01\x01\xa2\x01\x14\x12\x0fLENGTH_PREFIXED\x18\x84\x07\xb2\x01\x03\x08\xe8\x07R\x0fmessageEncoding\x12\x82\x01\n\x0bjson_format\x18\x06 \x01(\x0e\x32&.google.protobuf.FeatureSet.JsonFormatB9\x88\x01\x01\x98\x01\x03\x98\x01\x06\x98\x01\x01\xa2\x01\x17\x12\x12LEGACY_BEST_EFFORT\x18\x84\x07\xa2\x01\n\x12\x05\x41LLOW\x18\xe7\x07\xb2\x01\x03\x08\xe8\x07R\njsonFormat\x12\xab\x01\n\x14\x65nforce_naming_style\x18\x07 \x01(\x0e\x32..google.protobuf.FeatureSet.EnforceNamingStyleBI\x88\x01\x02\x98\x01\x01\x98\x01\x02\x98\x01\x03\x98\x01\x04\x98\x01\x05\x98\x01\x06\x98\x01\x07\x98\x01\x08\x98\x01\t\xa2\x01\x11\x12\x0cSTYLE_LEGACY\x18\x84\x07\xa2\x01\x0e\x12\tSTYLE2024\x18\xe9\x07\xb2\x01\x03\x08\xe9\x07R\x12\x65nforceNamingStyle\x12\xb9\x01\n\x19\x64\x65\x66\x61ult_symbol_visibility\x18\x08 \x01(\x0e\x32\x45.google.protobuf.FeatureSet.VisibilityFeature.DefaultSymbolVisibilityB6\x88\x01\x02\x98\x01\x01\xa2\x01\x0f\x12\nEXPORT_ALL\x18\x84\x07\xa2\x01\x15\x12\x10\x45XPORT_TOP_LEVEL\x18\xe9\x07\xb2\x01\x03\x08\xe9\x07R\x17\x64\x65\x66\x61ultSymbolVisibility\x1a\xa1\x01\n\x11VisibilityFeature\"\x81\x01\n\x17\x44\x65\x66\x61ultSymbolVisibility\x12%\n!DEFAULT_SYMBOL_VISIBILITY_UNKNOWN\x10\x00\x12\x0e\n\nEXPORT_ALL\x10\x01\x12\x14\n\x10\x45XPORT_TOP_LEVEL\x10\x02\x12\r\n\tLOCAL_ALL\x10\x03\x12\n\n\x06STRICT\x10\x04J\x08\x08\x01\x10\x80\x80\x80\x80\x02\"\\\n\rFieldPresence\x12\x1a\n\x16\x46IELD_PRESENCE_UNKNOWN\x10\x00\x12\x0c\n\x08\x45XPLICIT\x10\x01\x12\x0c\n\x08IMPLICIT\x10\x02\x12\x13\n\x0fLEGACY_REQUIRED\x10\x03\"7\n\x08\x45numType\x12\x15\n\x11\x45NUM_TYPE_UNKNOWN\x10\x00\x12\x08\n\x04OPEN\x10\x01\x12\n\n\x06\x43LOSED\x10\x02\"V\n\x15RepeatedFieldEncoding\x12#\n\x1fREPEATED_FIELD_ENCODING_UNKNOWN\x10\x00\x12\n\n\x06PACKED\x10\x01\x12\x0c\n\x08\x45XPANDED\x10\x02\"I\n\x0eUtf8Validation\x12\x1b\n\x17UTF8_VALIDATION_UNKNOWN\x10\x00\x12\n\n\x06VERIFY\x10\x02\x12\x08\n\x04NONE\x10\x03\"\x04\x08\x01\x10\x01\"S\n\x0fMessageEncoding\x12\x1c\n\x18MESSAGE_ENCODING_UNKNOWN\x10\x00\x12\x13\n\x0fLENGTH_PREFIXED\x10\x01\x12\r\n\tDELIMITED\x10\x02\"H\n\nJsonFormat\x12\x17\n\x13JSON_FORMAT_UNKNOWN\x10\x00\x12\t\n\x05\x41LLOW\x10\x01\x12\x16\n\x12LEGACY_BEST_EFFORT\x10\x02\"W\n\x12\x45nforceNamingStyle\x12 \n\x1c\x45NFORCE_NAMING_STYLE_UNKNOWN\x10\x00\x12\r\n\tSTYLE2024\x10\x01\x12\x10\n\x0cSTYLE_LEGACY\x10\x02*\x06\x08\xe8\x07\x10\x8bN*\x06\x08\x8bN\x10\x90N*\x06\x08\x90N\x10\x91NJ\x06\x08\xe7\x07\x10\xe8\x07\"\xef\x03\n\x12\x46\x65\x61tureSetDefaults\x12X\n\x08\x64\x65\x66\x61ults\x18\x01 \x03(\x0b\x32<.google.protobuf.FeatureSetDefaults.FeatureSetEditionDefaultR\x08\x64\x65\x66\x61ults\x12\x41\n\x0fminimum_edition\x18\x04 \x01(\x0e\x32\x18.google.protobuf.EditionR\x0eminimumEdition\x12\x41\n\x0fmaximum_edition\x18\x05 \x01(\x0e\x32\x18.google.protobuf.EditionR\x0emaximumEdition\x1a\xf8\x01\n\x18\x46\x65\x61tureSetEditionDefault\x12\x32\n\x07\x65\x64ition\x18\x03 \x01(\x0e\x32\x18.google.protobuf.EditionR\x07\x65\x64ition\x12N\n\x14overridable_features\x18\x04 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\x13overridableFeatures\x12\x42\n\x0e\x66ixed_features\x18\x05 \x01(\x0b\x32\x1b.google.protobuf.FeatureSetR\rfixedFeaturesJ\x04\x08\x01\x10\x02J\x04\x08\x02\x10\x03R\x08\x66\x65\x61tures\"\xb5\x02\n\x0eSourceCodeInfo\x12\x44\n\x08location\x18\x01 \x03(\x0b\x32(.google.protobuf.SourceCodeInfo.LocationR\x08location\x1a\xce\x01\n\x08Location\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x16\n\x04span\x18\x02 \x03(\x05\x42\x02\x10\x01R\x04span\x12)\n\x10leading_comments\x18\x03 \x01(\tR\x0fleadingComments\x12+\n\x11trailing_comments\x18\x04 \x01(\tR\x10trailingComments\x12:\n\x19leading_detached_comments\x18\x06 \x03(\tR\x17leadingDetachedComments*\x0c\x08\x80\xec\xca\xff\x01\x10\x81\xec\xca\xff\x01\"\xd0\x02\n\x11GeneratedCodeInfo\x12M\n\nannotation\x18\x01 \x03(\x0b\x32-.google.protobuf.GeneratedCodeInfo.AnnotationR\nannotation\x1a\xeb\x01\n\nAnnotation\x12\x16\n\x04path\x18\x01 \x03(\x05\x42\x02\x10\x01R\x04path\x12\x1f\n\x0bsource_file\x18\x02 \x01(\tR\nsourceFile\x12\x14\n\x05\x62\x65gin\x18\x03 \x01(\x05R\x05\x62\x65gin\x12\x10\n\x03\x65nd\x18\x04 \x01(\x05R\x03\x65nd\x12R\n\x08semantic\x18\x05 \x01(\x0e\x32\x36.google.protobuf.GeneratedCodeInfo.Annotation.SemanticR\x08semantic\"(\n\x08Semantic\x12\x08\n\x04NONE\x10\x00\x12\x07\n\x03SET\x10\x01\x12\t\n\x05\x41LIAS\x10\x02*\xbe\x02\n\x07\x45\x64ition\x12\x13\n\x0f\x45\x44ITION_UNKNOWN\x10\x00\x12\x13\n\x0e\x45\x44ITION_LEGACY\x10\x84\x07\x12\x13\n\x0e\x45\x44ITION_PROTO2\x10\xe6\x07\x12\x13\n\x0e\x45\x44ITION_PROTO3\x10\xe7\x07\x12\x11\n\x0c\x45\x44ITION_2023\x10\xe8\x07\x12\x11\n\x0c\x45\x44ITION_2024\x10\xe9\x07\x12\x15\n\x10\x45\x44ITION_UNSTABLE\x10\x8fN\x12\x17\n\x13\x45\x44ITION_1_TEST_ONLY\x10\x01\x12\x17\n\x13\x45\x44ITION_2_TEST_ONLY\x10\x02\x12\x1d\n\x17\x45\x44ITION_99997_TEST_ONLY\x10\x9d\x8d\x06\x12\x1d\n\x17\x45\x44ITION_99998_TEST_ONLY\x10\x9e\x8d\x06\x12\x1d\n\x17\x45\x44ITION_99999_TEST_ONLY\x10\x9f\x8d\x06\x12\x13\n\x0b\x45\x44ITION_MAX\x10\xff\xff\xff\xff\x07*U\n\x10SymbolVisibility\x12\x14\n\x10VISIBILITY_UNSET\x10\x00\x12\x14\n\x10VISIBILITY_LOCAL\x10\x01\x12\x15\n\x11VISIBILITY_EXPORT\x10\x02\x42~\n\x13\x63om.google.protobufB\x10\x44\x65scriptorProtosH\x01Z-google.golang.org/protobuf/types/descriptorpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1aGoogle.Protobuf.Reflection') + +_globals = globals() +if not _descriptor._USE_C_DESCRIPTORS: + _EDITION = _descriptor.EnumDescriptor( + name='Edition', + full_name='google.protobuf.Edition', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='EDITION_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_LEGACY', index=1, number=900, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_PROTO2', index=2, number=998, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_PROTO3', index=3, number=999, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_2023', index=4, number=1000, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_2024', index=5, number=1001, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_UNSTABLE', index=6, number=9999, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_1_TEST_ONLY', index=7, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_2_TEST_ONLY', index=8, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_99997_TEST_ONLY', index=9, number=99997, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_99998_TEST_ONLY', index=10, number=99998, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_99999_TEST_ONLY', index=11, number=99999, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EDITION_MAX', index=12, number=2147483647, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_EDITION) + + _SYMBOLVISIBILITY = _descriptor.EnumDescriptor( + name='SymbolVisibility', + full_name='google.protobuf.SymbolVisibility', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='VISIBILITY_UNSET', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='VISIBILITY_LOCAL', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='VISIBILITY_EXPORT', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_SYMBOLVISIBILITY) + + _EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE = _descriptor.EnumDescriptor( + name='VerificationState', + full_name='google.protobuf.ExtensionRangeOptions.VerificationState', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='DECLARATION', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='UNVERIFIED', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE) + + _FIELDDESCRIPTORPROTO_TYPE = _descriptor.EnumDescriptor( + name='Type', + full_name='google.protobuf.FieldDescriptorProto.Type', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='TYPE_DOUBLE', index=0, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_FLOAT', index=1, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_INT64', index=2, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_UINT64', index=3, number=4, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_INT32', index=4, number=5, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_FIXED64', index=5, number=6, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_FIXED32', index=6, number=7, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_BOOL', index=7, number=8, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_STRING', index=8, number=9, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_GROUP', index=9, number=10, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_MESSAGE', index=10, number=11, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_BYTES', index=11, number=12, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_UINT32', index=12, number=13, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_ENUM', index=13, number=14, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_SFIXED32', index=14, number=15, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_SFIXED64', index=15, number=16, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_SINT32', index=16, number=17, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TYPE_SINT64', index=17, number=18, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDDESCRIPTORPROTO_TYPE) + + _FIELDDESCRIPTORPROTO_LABEL = _descriptor.EnumDescriptor( + name='Label', + full_name='google.protobuf.FieldDescriptorProto.Label', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='LABEL_OPTIONAL', index=0, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='LABEL_REPEATED', index=1, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='LABEL_REQUIRED', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDDESCRIPTORPROTO_LABEL) + + _FILEOPTIONS_OPTIMIZEMODE = _descriptor.EnumDescriptor( + name='OptimizeMode', + full_name='google.protobuf.FileOptions.OptimizeMode', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='SPEED', index=0, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='CODE_SIZE', index=1, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='LITE_RUNTIME', index=2, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FILEOPTIONS_OPTIMIZEMODE) + + _FIELDOPTIONS_CTYPE = _descriptor.EnumDescriptor( + name='CType', + full_name='google.protobuf.FieldOptions.CType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='STRING', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='CORD', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='STRING_PIECE', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDOPTIONS_CTYPE) + + _FIELDOPTIONS_JSTYPE = _descriptor.EnumDescriptor( + name='JSType', + full_name='google.protobuf.FieldOptions.JSType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='JS_NORMAL', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='JS_STRING', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='JS_NUMBER', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDOPTIONS_JSTYPE) + + _FIELDOPTIONS_OPTIONRETENTION = _descriptor.EnumDescriptor( + name='OptionRetention', + full_name='google.protobuf.FieldOptions.OptionRetention', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='RETENTION_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='RETENTION_RUNTIME', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='RETENTION_SOURCE', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDOPTIONS_OPTIONRETENTION) + + _FIELDOPTIONS_OPTIONTARGETTYPE = _descriptor.EnumDescriptor( + name='OptionTargetType', + full_name='google.protobuf.FieldOptions.OptionTargetType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_FILE', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_EXTENSION_RANGE', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_MESSAGE', index=3, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_FIELD', index=4, number=4, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_ONEOF', index=5, number=5, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_ENUM', index=6, number=6, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_ENUM_ENTRY', index=7, number=7, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_SERVICE', index=8, number=8, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='TARGET_TYPE_METHOD', index=9, number=9, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FIELDOPTIONS_OPTIONTARGETTYPE) + + _METHODOPTIONS_IDEMPOTENCYLEVEL = _descriptor.EnumDescriptor( + name='IdempotencyLevel', + full_name='google.protobuf.MethodOptions.IdempotencyLevel', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='IDEMPOTENCY_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='NO_SIDE_EFFECTS', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='IDEMPOTENT', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_METHODOPTIONS_IDEMPOTENCYLEVEL) + + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY = _descriptor.EnumDescriptor( + name='DefaultSymbolVisibility', + full_name='google.protobuf.FeatureSet.VisibilityFeature.DefaultSymbolVisibility', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='DEFAULT_SYMBOL_VISIBILITY_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EXPORT_ALL', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EXPORT_TOP_LEVEL', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='LOCAL_ALL', index=3, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='STRICT', index=4, number=4, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY) + + _FEATURESET_FIELDPRESENCE = _descriptor.EnumDescriptor( + name='FieldPresence', + full_name='google.protobuf.FeatureSet.FieldPresence', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='FIELD_PRESENCE_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EXPLICIT', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='IMPLICIT', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='LEGACY_REQUIRED', index=3, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_FIELDPRESENCE) + + _FEATURESET_ENUMTYPE = _descriptor.EnumDescriptor( + name='EnumType', + full_name='google.protobuf.FeatureSet.EnumType', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='ENUM_TYPE_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='OPEN', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='CLOSED', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_ENUMTYPE) + + _FEATURESET_REPEATEDFIELDENCODING = _descriptor.EnumDescriptor( + name='RepeatedFieldEncoding', + full_name='google.protobuf.FeatureSet.RepeatedFieldEncoding', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='REPEATED_FIELD_ENCODING_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='PACKED', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='EXPANDED', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_REPEATEDFIELDENCODING) + + _FEATURESET_UTF8VALIDATION = _descriptor.EnumDescriptor( + name='Utf8Validation', + full_name='google.protobuf.FeatureSet.Utf8Validation', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='UTF8_VALIDATION_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='VERIFY', index=1, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='NONE', index=2, number=3, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_UTF8VALIDATION) + + _FEATURESET_MESSAGEENCODING = _descriptor.EnumDescriptor( + name='MessageEncoding', + full_name='google.protobuf.FeatureSet.MessageEncoding', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='MESSAGE_ENCODING_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='LENGTH_PREFIXED', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='DELIMITED', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_MESSAGEENCODING) + + _FEATURESET_JSONFORMAT = _descriptor.EnumDescriptor( + name='JsonFormat', + full_name='google.protobuf.FeatureSet.JsonFormat', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='JSON_FORMAT_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='ALLOW', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='LEGACY_BEST_EFFORT', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_JSONFORMAT) + + _FEATURESET_ENFORCENAMINGSTYLE = _descriptor.EnumDescriptor( + name='EnforceNamingStyle', + full_name='google.protobuf.FeatureSet.EnforceNamingStyle', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='ENFORCE_NAMING_STYLE_UNKNOWN', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='STYLE2024', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='STYLE_LEGACY', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_FEATURESET_ENFORCENAMINGSTYLE) + + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC = _descriptor.EnumDescriptor( + name='Semantic', + full_name='google.protobuf.GeneratedCodeInfo.Annotation.Semantic', + filename=None, + file=DESCRIPTOR, + create_key=_descriptor._internal_create_key, + values=[ + _descriptor.EnumValueDescriptor( + name='NONE', index=0, number=0, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='SET', index=1, number=1, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + _descriptor.EnumValueDescriptor( + name='ALIAS', index=2, number=2, + serialized_options=None, + type=None, + create_key=_descriptor._internal_create_key), + ], + containing_type=None, + serialized_options=None, + ) + _sym_db.RegisterEnumDescriptor(_GENERATEDCODEINFO_ANNOTATION_SEMANTIC) + + + _FILEDESCRIPTORSET = _descriptor.Descriptor( + name='FileDescriptorSet', + full_name='google.protobuf.FileDescriptorSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='file', full_name='google.protobuf.FileDescriptorSet.file', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='file', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(536000000, 536000001), ], + oneofs=[ + ], + ) + + + _FILEDESCRIPTORPROTO = _descriptor.Descriptor( + name='FileDescriptorProto', + full_name='google.protobuf.FileDescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.FileDescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='package', full_name='google.protobuf.FileDescriptorProto.package', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='package', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='dependency', full_name='google.protobuf.FileDescriptorProto.dependency', index=2, + number=3, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='dependency', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='public_dependency', full_name='google.protobuf.FileDescriptorProto.public_dependency', index=3, + number=10, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='publicDependency', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='weak_dependency', full_name='google.protobuf.FileDescriptorProto.weak_dependency', index=4, + number=11, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='weakDependency', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='option_dependency', full_name='google.protobuf.FileDescriptorProto.option_dependency', index=5, + number=15, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='optionDependency', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='message_type', full_name='google.protobuf.FileDescriptorProto.message_type', index=6, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='messageType', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='enum_type', full_name='google.protobuf.FileDescriptorProto.enum_type', index=7, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='enumType', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='service', full_name='google.protobuf.FileDescriptorProto.service', index=8, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='service', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='extension', full_name='google.protobuf.FileDescriptorProto.extension', index=9, + number=7, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='extension', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.FileDescriptorProto.options', index=10, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='source_code_info', full_name='google.protobuf.FileDescriptorProto.source_code_info', index=11, + number=9, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='sourceCodeInfo', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='syntax', full_name='google.protobuf.FileDescriptorProto.syntax', index=12, + number=12, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='syntax', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='edition', full_name='google.protobuf.FileDescriptorProto.edition', index=13, + number=14, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='edition', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _DESCRIPTORPROTO_EXTENSIONRANGE = _descriptor.Descriptor( + name='ExtensionRange', + full_name='google.protobuf.DescriptorProto.ExtensionRange', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='start', full_name='google.protobuf.DescriptorProto.ExtensionRange.start', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='start', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='end', full_name='google.protobuf.DescriptorProto.ExtensionRange.end', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='end', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.DescriptorProto.ExtensionRange.options', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _DESCRIPTORPROTO_RESERVEDRANGE = _descriptor.Descriptor( + name='ReservedRange', + full_name='google.protobuf.DescriptorProto.ReservedRange', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='start', full_name='google.protobuf.DescriptorProto.ReservedRange.start', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='start', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='end', full_name='google.protobuf.DescriptorProto.ReservedRange.end', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='end', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _DESCRIPTORPROTO = _descriptor.Descriptor( + name='DescriptorProto', + full_name='google.protobuf.DescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.DescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='field', full_name='google.protobuf.DescriptorProto.field', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='field', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='extension', full_name='google.protobuf.DescriptorProto.extension', index=2, + number=6, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='extension', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='nested_type', full_name='google.protobuf.DescriptorProto.nested_type', index=3, + number=3, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='nestedType', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='enum_type', full_name='google.protobuf.DescriptorProto.enum_type', index=4, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='enumType', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='extension_range', full_name='google.protobuf.DescriptorProto.extension_range', index=5, + number=5, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='extensionRange', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='oneof_decl', full_name='google.protobuf.DescriptorProto.oneof_decl', index=6, + number=8, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='oneofDecl', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.DescriptorProto.options', index=7, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='reserved_range', full_name='google.protobuf.DescriptorProto.reserved_range', index=8, + number=9, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='reservedRange', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='reserved_name', full_name='google.protobuf.DescriptorProto.reserved_name', index=9, + number=10, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='reservedName', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='visibility', full_name='google.protobuf.DescriptorProto.visibility', index=10, + number=11, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='visibility', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_DESCRIPTORPROTO_EXTENSIONRANGE, _DESCRIPTORPROTO_RESERVEDRANGE, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _EXTENSIONRANGEOPTIONS_DECLARATION = _descriptor.Descriptor( + name='Declaration', + full_name='google.protobuf.ExtensionRangeOptions.Declaration', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='number', full_name='google.protobuf.ExtensionRangeOptions.Declaration.number', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='number', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='full_name', full_name='google.protobuf.ExtensionRangeOptions.Declaration.full_name', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='fullName', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='type', full_name='google.protobuf.ExtensionRangeOptions.Declaration.type', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='type', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='reserved', full_name='google.protobuf.ExtensionRangeOptions.Declaration.reserved', index=3, + number=5, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='reserved', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='repeated', full_name='google.protobuf.ExtensionRangeOptions.Declaration.repeated', index=4, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='repeated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _EXTENSIONRANGEOPTIONS = _descriptor.Descriptor( + name='ExtensionRangeOptions', + full_name='google.protobuf.ExtensionRangeOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.ExtensionRangeOptions.uninterpreted_option', index=0, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='declaration', full_name='google.protobuf.ExtensionRangeOptions.declaration', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\002', json_name='declaration', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.ExtensionRangeOptions.features', index=2, + number=50, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='verification', full_name='google.protobuf.ExtensionRangeOptions.verification', index=3, + number=3, type=14, cpp_type=8, label=1, + has_default_value=True, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\002', json_name='verification', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_EXTENSIONRANGEOPTIONS_DECLARATION, ], + enum_types=[ + _EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE, + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _FIELDDESCRIPTORPROTO = _descriptor.Descriptor( + name='FieldDescriptorProto', + full_name='google.protobuf.FieldDescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.FieldDescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='number', full_name='google.protobuf.FieldDescriptorProto.number', index=1, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='number', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='label', full_name='google.protobuf.FieldDescriptorProto.label', index=2, + number=4, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='label', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='type', full_name='google.protobuf.FieldDescriptorProto.type', index=3, + number=5, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='type', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='type_name', full_name='google.protobuf.FieldDescriptorProto.type_name', index=4, + number=6, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='typeName', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='extendee', full_name='google.protobuf.FieldDescriptorProto.extendee', index=5, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='extendee', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='default_value', full_name='google.protobuf.FieldDescriptorProto.default_value', index=6, + number=7, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='defaultValue', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='oneof_index', full_name='google.protobuf.FieldDescriptorProto.oneof_index', index=7, + number=9, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='oneofIndex', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='json_name', full_name='google.protobuf.FieldDescriptorProto.json_name', index=8, + number=10, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='jsonName', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.FieldDescriptorProto.options', index=9, + number=8, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='proto3_optional', full_name='google.protobuf.FieldDescriptorProto.proto3_optional', index=10, + number=17, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='proto3Optional', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _FIELDDESCRIPTORPROTO_TYPE, + _FIELDDESCRIPTORPROTO_LABEL, + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _ONEOFDESCRIPTORPROTO = _descriptor.Descriptor( + name='OneofDescriptorProto', + full_name='google.protobuf.OneofDescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.OneofDescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.OneofDescriptorProto.options', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE = _descriptor.Descriptor( + name='EnumReservedRange', + full_name='google.protobuf.EnumDescriptorProto.EnumReservedRange', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='start', full_name='google.protobuf.EnumDescriptorProto.EnumReservedRange.start', index=0, + number=1, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='start', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='end', full_name='google.protobuf.EnumDescriptorProto.EnumReservedRange.end', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='end', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _ENUMDESCRIPTORPROTO = _descriptor.Descriptor( + name='EnumDescriptorProto', + full_name='google.protobuf.EnumDescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.EnumDescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='google.protobuf.EnumDescriptorProto.value', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='value', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.EnumDescriptorProto.options', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='reserved_range', full_name='google.protobuf.EnumDescriptorProto.reserved_range', index=3, + number=4, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='reservedRange', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='reserved_name', full_name='google.protobuf.EnumDescriptorProto.reserved_name', index=4, + number=5, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='reservedName', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='visibility', full_name='google.protobuf.EnumDescriptorProto.visibility', index=5, + number=6, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='visibility', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _ENUMVALUEDESCRIPTORPROTO = _descriptor.Descriptor( + name='EnumValueDescriptorProto', + full_name='google.protobuf.EnumValueDescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.EnumValueDescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='number', full_name='google.protobuf.EnumValueDescriptorProto.number', index=1, + number=2, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='number', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.EnumValueDescriptorProto.options', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _SERVICEDESCRIPTORPROTO = _descriptor.Descriptor( + name='ServiceDescriptorProto', + full_name='google.protobuf.ServiceDescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.ServiceDescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='method', full_name='google.protobuf.ServiceDescriptorProto.method', index=1, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='method', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.ServiceDescriptorProto.options', index=2, + number=3, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _METHODDESCRIPTORPROTO = _descriptor.Descriptor( + name='MethodDescriptorProto', + full_name='google.protobuf.MethodDescriptorProto', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.MethodDescriptorProto.name', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='input_type', full_name='google.protobuf.MethodDescriptorProto.input_type', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='inputType', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='output_type', full_name='google.protobuf.MethodDescriptorProto.output_type', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='outputType', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='options', full_name='google.protobuf.MethodDescriptorProto.options', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='options', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='client_streaming', full_name='google.protobuf.MethodDescriptorProto.client_streaming', index=4, + number=5, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='clientStreaming', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='server_streaming', full_name='google.protobuf.MethodDescriptorProto.server_streaming', index=5, + number=6, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='serverStreaming', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _FILEOPTIONS = _descriptor.Descriptor( + name='FileOptions', + full_name='google.protobuf.FileOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='java_package', full_name='google.protobuf.FileOptions.java_package', index=0, + number=1, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='javaPackage', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='java_outer_classname', full_name='google.protobuf.FileOptions.java_outer_classname', index=1, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='javaOuterClassname', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='java_multiple_files', full_name='google.protobuf.FileOptions.java_multiple_files', index=2, + number=10, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='javaMultipleFiles', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='java_generate_equals_and_hash', full_name='google.protobuf.FileOptions.java_generate_equals_and_hash', index=3, + number=20, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\030\001', json_name='javaGenerateEqualsAndHash', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='java_string_check_utf8', full_name='google.protobuf.FileOptions.java_string_check_utf8', index=4, + number=27, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='javaStringCheckUtf8', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='optimize_for', full_name='google.protobuf.FileOptions.optimize_for', index=5, + number=9, type=14, cpp_type=8, label=1, + has_default_value=True, default_value=1, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='optimizeFor', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='go_package', full_name='google.protobuf.FileOptions.go_package', index=6, + number=11, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='goPackage', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='cc_generic_services', full_name='google.protobuf.FileOptions.cc_generic_services', index=7, + number=16, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='ccGenericServices', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='java_generic_services', full_name='google.protobuf.FileOptions.java_generic_services', index=8, + number=17, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='javaGenericServices', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='py_generic_services', full_name='google.protobuf.FileOptions.py_generic_services', index=9, + number=18, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='pyGenericServices', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecated', full_name='google.protobuf.FileOptions.deprecated', index=10, + number=23, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='cc_enable_arenas', full_name='google.protobuf.FileOptions.cc_enable_arenas', index=11, + number=31, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=True, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='ccEnableArenas', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='objc_class_prefix', full_name='google.protobuf.FileOptions.objc_class_prefix', index=12, + number=36, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='objcClassPrefix', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='csharp_namespace', full_name='google.protobuf.FileOptions.csharp_namespace', index=13, + number=37, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='csharpNamespace', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='swift_prefix', full_name='google.protobuf.FileOptions.swift_prefix', index=14, + number=39, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='swiftPrefix', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='php_class_prefix', full_name='google.protobuf.FileOptions.php_class_prefix', index=15, + number=40, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='phpClassPrefix', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='php_namespace', full_name='google.protobuf.FileOptions.php_namespace', index=16, + number=41, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='phpNamespace', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='php_metadata_namespace', full_name='google.protobuf.FileOptions.php_metadata_namespace', index=17, + number=44, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='phpMetadataNamespace', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='ruby_package', full_name='google.protobuf.FileOptions.ruby_package', index=18, + number=45, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='rubyPackage', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.FileOptions.features', index=19, + number=50, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.FileOptions.uninterpreted_option', index=20, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _FILEOPTIONS_OPTIMIZEMODE, + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _MESSAGEOPTIONS = _descriptor.Descriptor( + name='MessageOptions', + full_name='google.protobuf.MessageOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='message_set_wire_format', full_name='google.protobuf.MessageOptions.message_set_wire_format', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='messageSetWireFormat', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='no_standard_descriptor_accessor', full_name='google.protobuf.MessageOptions.no_standard_descriptor_accessor', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='noStandardDescriptorAccessor', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecated', full_name='google.protobuf.MessageOptions.deprecated', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='map_entry', full_name='google.protobuf.MessageOptions.map_entry', index=3, + number=7, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='mapEntry', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecated_legacy_json_field_conflicts', full_name='google.protobuf.MessageOptions.deprecated_legacy_json_field_conflicts', index=4, + number=11, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\030\001', json_name='deprecatedLegacyJsonFieldConflicts', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.MessageOptions.features', index=5, + number=12, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.MessageOptions.uninterpreted_option', index=6, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _FIELDOPTIONS_EDITIONDEFAULT = _descriptor.Descriptor( + name='EditionDefault', + full_name='google.protobuf.FieldOptions.EditionDefault', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='edition', full_name='google.protobuf.FieldOptions.EditionDefault.edition', index=0, + number=3, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='edition', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='value', full_name='google.protobuf.FieldOptions.EditionDefault.value', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='value', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _FIELDOPTIONS_FEATURESUPPORT = _descriptor.Descriptor( + name='FeatureSupport', + full_name='google.protobuf.FieldOptions.FeatureSupport', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='edition_introduced', full_name='google.protobuf.FieldOptions.FeatureSupport.edition_introduced', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='editionIntroduced', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='edition_deprecated', full_name='google.protobuf.FieldOptions.FeatureSupport.edition_deprecated', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='editionDeprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecation_warning', full_name='google.protobuf.FieldOptions.FeatureSupport.deprecation_warning', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecationWarning', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='edition_removed', full_name='google.protobuf.FieldOptions.FeatureSupport.edition_removed', index=3, + number=4, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='editionRemoved', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _FIELDOPTIONS = _descriptor.Descriptor( + name='FieldOptions', + full_name='google.protobuf.FieldOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='ctype', full_name='google.protobuf.FieldOptions.ctype', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=True, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='ctype', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='packed', full_name='google.protobuf.FieldOptions.packed', index=1, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='packed', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='jstype', full_name='google.protobuf.FieldOptions.jstype', index=2, + number=6, type=14, cpp_type=8, label=1, + has_default_value=True, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='jstype', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='lazy', full_name='google.protobuf.FieldOptions.lazy', index=3, + number=5, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='lazy', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='unverified_lazy', full_name='google.protobuf.FieldOptions.unverified_lazy', index=4, + number=15, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='unverifiedLazy', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecated', full_name='google.protobuf.FieldOptions.deprecated', index=5, + number=3, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='weak', full_name='google.protobuf.FieldOptions.weak', index=6, + number=10, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\030\001', json_name='weak', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='debug_redact', full_name='google.protobuf.FieldOptions.debug_redact', index=7, + number=16, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='debugRedact', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='retention', full_name='google.protobuf.FieldOptions.retention', index=8, + number=17, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='retention', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='targets', full_name='google.protobuf.FieldOptions.targets', index=9, + number=19, type=14, cpp_type=8, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='targets', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='edition_defaults', full_name='google.protobuf.FieldOptions.edition_defaults', index=10, + number=20, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='editionDefaults', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.FieldOptions.features', index=11, + number=21, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='feature_support', full_name='google.protobuf.FieldOptions.feature_support', index=12, + number=22, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='featureSupport', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.FieldOptions.uninterpreted_option', index=13, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_FIELDOPTIONS_EDITIONDEFAULT, _FIELDOPTIONS_FEATURESUPPORT, ], + enum_types=[ + _FIELDOPTIONS_CTYPE, + _FIELDOPTIONS_JSTYPE, + _FIELDOPTIONS_OPTIONRETENTION, + _FIELDOPTIONS_OPTIONTARGETTYPE, + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _ONEOFOPTIONS = _descriptor.Descriptor( + name='OneofOptions', + full_name='google.protobuf.OneofOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.OneofOptions.features', index=0, + number=1, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.OneofOptions.uninterpreted_option', index=1, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _ENUMOPTIONS = _descriptor.Descriptor( + name='EnumOptions', + full_name='google.protobuf.EnumOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='allow_alias', full_name='google.protobuf.EnumOptions.allow_alias', index=0, + number=2, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='allowAlias', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecated', full_name='google.protobuf.EnumOptions.deprecated', index=1, + number=3, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecated_legacy_json_field_conflicts', full_name='google.protobuf.EnumOptions.deprecated_legacy_json_field_conflicts', index=2, + number=6, type=8, cpp_type=7, label=1, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\030\001', json_name='deprecatedLegacyJsonFieldConflicts', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.EnumOptions.features', index=3, + number=7, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.EnumOptions.uninterpreted_option', index=4, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _ENUMVALUEOPTIONS = _descriptor.Descriptor( + name='EnumValueOptions', + full_name='google.protobuf.EnumValueOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='deprecated', full_name='google.protobuf.EnumValueOptions.deprecated', index=0, + number=1, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.EnumValueOptions.features', index=1, + number=2, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='debug_redact', full_name='google.protobuf.EnumValueOptions.debug_redact', index=2, + number=3, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='debugRedact', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='feature_support', full_name='google.protobuf.EnumValueOptions.feature_support', index=3, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='featureSupport', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.EnumValueOptions.uninterpreted_option', index=4, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _SERVICEOPTIONS = _descriptor.Descriptor( + name='ServiceOptions', + full_name='google.protobuf.ServiceOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.ServiceOptions.features', index=0, + number=34, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='deprecated', full_name='google.protobuf.ServiceOptions.deprecated', index=1, + number=33, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.ServiceOptions.uninterpreted_option', index=2, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _METHODOPTIONS = _descriptor.Descriptor( + name='MethodOptions', + full_name='google.protobuf.MethodOptions', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='deprecated', full_name='google.protobuf.MethodOptions.deprecated', index=0, + number=33, type=8, cpp_type=7, label=1, + has_default_value=True, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='deprecated', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='idempotency_level', full_name='google.protobuf.MethodOptions.idempotency_level', index=1, + number=34, type=14, cpp_type=8, label=1, + has_default_value=True, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='idempotencyLevel', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='features', full_name='google.protobuf.MethodOptions.features', index=2, + number=35, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='features', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='uninterpreted_option', full_name='google.protobuf.MethodOptions.uninterpreted_option', index=3, + number=999, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='uninterpretedOption', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _METHODOPTIONS_IDEMPOTENCYLEVEL, + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 536870912), ], + oneofs=[ + ], + ) + + + _UNINTERPRETEDOPTION_NAMEPART = _descriptor.Descriptor( + name='NamePart', + full_name='google.protobuf.UninterpretedOption.NamePart', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name_part', full_name='google.protobuf.UninterpretedOption.NamePart.name_part', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='namePart', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='is_extension', full_name='google.protobuf.UninterpretedOption.NamePart.is_extension', index=1, + number=2, type=8, cpp_type=7, label=2, + has_default_value=False, default_value=False, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='isExtension', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _UNINTERPRETEDOPTION = _descriptor.Descriptor( + name='UninterpretedOption', + full_name='google.protobuf.UninterpretedOption', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='name', full_name='google.protobuf.UninterpretedOption.name', index=0, + number=2, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='name', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='identifier_value', full_name='google.protobuf.UninterpretedOption.identifier_value', index=1, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='identifierValue', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='positive_int_value', full_name='google.protobuf.UninterpretedOption.positive_int_value', index=2, + number=4, type=4, cpp_type=4, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='positiveIntValue', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='negative_int_value', full_name='google.protobuf.UninterpretedOption.negative_int_value', index=3, + number=5, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='negativeIntValue', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='double_value', full_name='google.protobuf.UninterpretedOption.double_value', index=4, + number=6, type=1, cpp_type=5, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='doubleValue', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='string_value', full_name='google.protobuf.UninterpretedOption.string_value', index=5, + number=7, type=12, cpp_type=9, label=1, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='stringValue', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='aggregate_value', full_name='google.protobuf.UninterpretedOption.aggregate_value', index=6, + number=8, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='aggregateValue', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_UNINTERPRETEDOPTION_NAMEPART, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _FEATURESET_VISIBILITYFEATURE = _descriptor.Descriptor( + name='VisibilityFeature', + full_name='google.protobuf.FeatureSet.VisibilityFeature', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY, + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _FEATURESET = _descriptor.Descriptor( + name='FeatureSet', + full_name='google.protobuf.FeatureSet', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='field_presence', full_name='google.protobuf.FeatureSet.field_presence', index=0, + number=1, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\001\230\001\004\230\001\001\242\001\r\022\010EXPLICIT\030\204\007\242\001\r\022\010IMPLICIT\030\347\007\242\001\r\022\010EXPLICIT\030\350\007\262\001\003\010\350\007', json_name='fieldPresence', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='enum_type', full_name='google.protobuf.FeatureSet.enum_type', index=1, + number=2, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\001\230\001\006\230\001\001\242\001\013\022\006CLOSED\030\204\007\242\001\t\022\004OPEN\030\347\007\262\001\003\010\350\007', json_name='enumType', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='repeated_field_encoding', full_name='google.protobuf.FeatureSet.repeated_field_encoding', index=2, + number=3, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\001\230\001\004\230\001\001\242\001\r\022\010EXPANDED\030\204\007\242\001\013\022\006PACKED\030\347\007\262\001\003\010\350\007', json_name='repeatedFieldEncoding', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='utf8_validation', full_name='google.protobuf.FeatureSet.utf8_validation', index=3, + number=4, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\001\230\001\004\230\001\001\242\001\t\022\004NONE\030\204\007\242\001\013\022\006VERIFY\030\347\007\262\001\003\010\350\007', json_name='utf8Validation', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='message_encoding', full_name='google.protobuf.FeatureSet.message_encoding', index=4, + number=5, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\001\230\001\004\230\001\001\242\001\024\022\017LENGTH_PREFIXED\030\204\007\262\001\003\010\350\007', json_name='messageEncoding', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='json_format', full_name='google.protobuf.FeatureSet.json_format', index=5, + number=6, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\001\230\001\003\230\001\006\230\001\001\242\001\027\022\022LEGACY_BEST_EFFORT\030\204\007\242\001\n\022\005ALLOW\030\347\007\262\001\003\010\350\007', json_name='jsonFormat', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='enforce_naming_style', full_name='google.protobuf.FeatureSet.enforce_naming_style', index=6, + number=7, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\002\230\001\001\230\001\002\230\001\003\230\001\004\230\001\005\230\001\006\230\001\007\230\001\010\230\001\t\242\001\021\022\014STYLE_LEGACY\030\204\007\242\001\016\022\tSTYLE2024\030\351\007\262\001\003\010\351\007', json_name='enforceNamingStyle', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='default_symbol_visibility', full_name='google.protobuf.FeatureSet.default_symbol_visibility', index=7, + number=8, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\210\001\002\230\001\001\242\001\017\022\nEXPORT_ALL\030\204\007\242\001\025\022\020EXPORT_TOP_LEVEL\030\351\007\262\001\003\010\351\007', json_name='defaultSymbolVisibility', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_FEATURESET_VISIBILITYFEATURE, ], + enum_types=[ + _FEATURESET_FIELDPRESENCE, + _FEATURESET_ENUMTYPE, + _FEATURESET_REPEATEDFIELDENCODING, + _FEATURESET_UTF8VALIDATION, + _FEATURESET_MESSAGEENCODING, + _FEATURESET_JSONFORMAT, + _FEATURESET_ENFORCENAMINGSTYLE, + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(1000, 9995), (9995, 10000), (10000, 10001), ], + oneofs=[ + ], + ) + + + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT = _descriptor.Descriptor( + name='FeatureSetEditionDefault', + full_name='google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='edition', full_name='google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault.edition', index=0, + number=3, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='edition', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='overridable_features', full_name='google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault.overridable_features', index=1, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='overridableFeatures', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='fixed_features', full_name='google.protobuf.FeatureSetDefaults.FeatureSetEditionDefault.fixed_features', index=2, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='fixedFeatures', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _FEATURESETDEFAULTS = _descriptor.Descriptor( + name='FeatureSetDefaults', + full_name='google.protobuf.FeatureSetDefaults', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='defaults', full_name='google.protobuf.FeatureSetDefaults.defaults', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='defaults', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='minimum_edition', full_name='google.protobuf.FeatureSetDefaults.minimum_edition', index=1, + number=4, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='minimumEdition', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='maximum_edition', full_name='google.protobuf.FeatureSetDefaults.maximum_edition', index=2, + number=5, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='maximumEdition', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + + _SOURCECODEINFO_LOCATION = _descriptor.Descriptor( + name='Location', + full_name='google.protobuf.SourceCodeInfo.Location', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='path', full_name='google.protobuf.SourceCodeInfo.Location.path', index=0, + number=1, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', json_name='path', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='span', full_name='google.protobuf.SourceCodeInfo.Location.span', index=1, + number=2, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', json_name='span', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='leading_comments', full_name='google.protobuf.SourceCodeInfo.Location.leading_comments', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='leadingComments', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='trailing_comments', full_name='google.protobuf.SourceCodeInfo.Location.trailing_comments', index=3, + number=4, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='trailingComments', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='leading_detached_comments', full_name='google.protobuf.SourceCodeInfo.Location.leading_detached_comments', index=4, + number=6, type=9, cpp_type=9, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='leadingDetachedComments', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _SOURCECODEINFO = _descriptor.Descriptor( + name='SourceCodeInfo', + full_name='google.protobuf.SourceCodeInfo', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='location', full_name='google.protobuf.SourceCodeInfo.location', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='location', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_SOURCECODEINFO_LOCATION, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=True, + extension_ranges=[(536000000, 536000001), ], + oneofs=[ + ], + ) + + + _GENERATEDCODEINFO_ANNOTATION = _descriptor.Descriptor( + name='Annotation', + full_name='google.protobuf.GeneratedCodeInfo.Annotation', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='path', full_name='google.protobuf.GeneratedCodeInfo.Annotation.path', index=0, + number=1, type=5, cpp_type=1, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=b'\020\001', json_name='path', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='source_file', full_name='google.protobuf.GeneratedCodeInfo.Annotation.source_file', index=1, + number=2, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='sourceFile', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='begin', full_name='google.protobuf.GeneratedCodeInfo.Annotation.begin', index=2, + number=3, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='begin', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='end', full_name='google.protobuf.GeneratedCodeInfo.Annotation.end', index=3, + number=4, type=5, cpp_type=1, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='end', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='semantic', full_name='google.protobuf.GeneratedCodeInfo.Annotation.semantic', index=4, + number=5, type=14, cpp_type=8, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='semantic', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC, + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _GENERATEDCODEINFO = _descriptor.Descriptor( + name='GeneratedCodeInfo', + full_name='google.protobuf.GeneratedCodeInfo', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='annotation', full_name='google.protobuf.GeneratedCodeInfo.annotation', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, json_name='annotation', file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_GENERATEDCODEINFO_ANNOTATION, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + extension_ranges=[], + oneofs=[ + ], + ) + + _FILEDESCRIPTORSET.fields_by_name['file'].message_type = _FILEDESCRIPTORPROTO + _FILEDESCRIPTORPROTO.fields_by_name['message_type'].message_type = _DESCRIPTORPROTO + _FILEDESCRIPTORPROTO.fields_by_name['enum_type'].message_type = _ENUMDESCRIPTORPROTO + _FILEDESCRIPTORPROTO.fields_by_name['service'].message_type = _SERVICEDESCRIPTORPROTO + _FILEDESCRIPTORPROTO.fields_by_name['extension'].message_type = _FIELDDESCRIPTORPROTO + _FILEDESCRIPTORPROTO.fields_by_name['options'].message_type = _FILEOPTIONS + _FILEDESCRIPTORPROTO.fields_by_name['source_code_info'].message_type = _SOURCECODEINFO + _FILEDESCRIPTORPROTO.fields_by_name['edition'].enum_type = _EDITION + _DESCRIPTORPROTO_EXTENSIONRANGE.fields_by_name['options'].message_type = _EXTENSIONRANGEOPTIONS + _DESCRIPTORPROTO_EXTENSIONRANGE.containing_type = _DESCRIPTORPROTO + _DESCRIPTORPROTO_RESERVEDRANGE.containing_type = _DESCRIPTORPROTO + _DESCRIPTORPROTO.fields_by_name['field'].message_type = _FIELDDESCRIPTORPROTO + _DESCRIPTORPROTO.fields_by_name['extension'].message_type = _FIELDDESCRIPTORPROTO + _DESCRIPTORPROTO.fields_by_name['nested_type'].message_type = _DESCRIPTORPROTO + _DESCRIPTORPROTO.fields_by_name['enum_type'].message_type = _ENUMDESCRIPTORPROTO + _DESCRIPTORPROTO.fields_by_name['extension_range'].message_type = _DESCRIPTORPROTO_EXTENSIONRANGE + _DESCRIPTORPROTO.fields_by_name['oneof_decl'].message_type = _ONEOFDESCRIPTORPROTO + _DESCRIPTORPROTO.fields_by_name['options'].message_type = _MESSAGEOPTIONS + _DESCRIPTORPROTO.fields_by_name['reserved_range'].message_type = _DESCRIPTORPROTO_RESERVEDRANGE + _DESCRIPTORPROTO.fields_by_name['visibility'].enum_type = _SYMBOLVISIBILITY + _EXTENSIONRANGEOPTIONS_DECLARATION.containing_type = _EXTENSIONRANGEOPTIONS + _EXTENSIONRANGEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _EXTENSIONRANGEOPTIONS.fields_by_name['declaration'].message_type = _EXTENSIONRANGEOPTIONS_DECLARATION + _EXTENSIONRANGEOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _EXTENSIONRANGEOPTIONS.fields_by_name['verification'].enum_type = _EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE + _EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE.containing_type = _EXTENSIONRANGEOPTIONS + _FIELDDESCRIPTORPROTO.fields_by_name['label'].enum_type = _FIELDDESCRIPTORPROTO_LABEL + _FIELDDESCRIPTORPROTO.fields_by_name['type'].enum_type = _FIELDDESCRIPTORPROTO_TYPE + _FIELDDESCRIPTORPROTO.fields_by_name['options'].message_type = _FIELDOPTIONS + _FIELDDESCRIPTORPROTO_TYPE.containing_type = _FIELDDESCRIPTORPROTO + _FIELDDESCRIPTORPROTO_LABEL.containing_type = _FIELDDESCRIPTORPROTO + _ONEOFDESCRIPTORPROTO.fields_by_name['options'].message_type = _ONEOFOPTIONS + _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE.containing_type = _ENUMDESCRIPTORPROTO + _ENUMDESCRIPTORPROTO.fields_by_name['value'].message_type = _ENUMVALUEDESCRIPTORPROTO + _ENUMDESCRIPTORPROTO.fields_by_name['options'].message_type = _ENUMOPTIONS + _ENUMDESCRIPTORPROTO.fields_by_name['reserved_range'].message_type = _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE + _ENUMDESCRIPTORPROTO.fields_by_name['visibility'].enum_type = _SYMBOLVISIBILITY + _ENUMVALUEDESCRIPTORPROTO.fields_by_name['options'].message_type = _ENUMVALUEOPTIONS + _SERVICEDESCRIPTORPROTO.fields_by_name['method'].message_type = _METHODDESCRIPTORPROTO + _SERVICEDESCRIPTORPROTO.fields_by_name['options'].message_type = _SERVICEOPTIONS + _METHODDESCRIPTORPROTO.fields_by_name['options'].message_type = _METHODOPTIONS + _FILEOPTIONS.fields_by_name['optimize_for'].enum_type = _FILEOPTIONS_OPTIMIZEMODE + _FILEOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _FILEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _FILEOPTIONS_OPTIMIZEMODE.containing_type = _FILEOPTIONS + _MESSAGEOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _MESSAGEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _FIELDOPTIONS_EDITIONDEFAULT.fields_by_name['edition'].enum_type = _EDITION + _FIELDOPTIONS_EDITIONDEFAULT.containing_type = _FIELDOPTIONS + _FIELDOPTIONS_FEATURESUPPORT.fields_by_name['edition_introduced'].enum_type = _EDITION + _FIELDOPTIONS_FEATURESUPPORT.fields_by_name['edition_deprecated'].enum_type = _EDITION + _FIELDOPTIONS_FEATURESUPPORT.fields_by_name['edition_removed'].enum_type = _EDITION + _FIELDOPTIONS_FEATURESUPPORT.containing_type = _FIELDOPTIONS + _FIELDOPTIONS.fields_by_name['ctype'].enum_type = _FIELDOPTIONS_CTYPE + _FIELDOPTIONS.fields_by_name['jstype'].enum_type = _FIELDOPTIONS_JSTYPE + _FIELDOPTIONS.fields_by_name['retention'].enum_type = _FIELDOPTIONS_OPTIONRETENTION + _FIELDOPTIONS.fields_by_name['targets'].enum_type = _FIELDOPTIONS_OPTIONTARGETTYPE + _FIELDOPTIONS.fields_by_name['edition_defaults'].message_type = _FIELDOPTIONS_EDITIONDEFAULT + _FIELDOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _FIELDOPTIONS.fields_by_name['feature_support'].message_type = _FIELDOPTIONS_FEATURESUPPORT + _FIELDOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _FIELDOPTIONS_CTYPE.containing_type = _FIELDOPTIONS + _FIELDOPTIONS_JSTYPE.containing_type = _FIELDOPTIONS + _FIELDOPTIONS_OPTIONRETENTION.containing_type = _FIELDOPTIONS + _FIELDOPTIONS_OPTIONTARGETTYPE.containing_type = _FIELDOPTIONS + _ONEOFOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _ONEOFOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _ENUMOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _ENUMOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _ENUMVALUEOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _ENUMVALUEOPTIONS.fields_by_name['feature_support'].message_type = _FIELDOPTIONS_FEATURESUPPORT + _ENUMVALUEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _SERVICEOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _SERVICEOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _METHODOPTIONS.fields_by_name['idempotency_level'].enum_type = _METHODOPTIONS_IDEMPOTENCYLEVEL + _METHODOPTIONS.fields_by_name['features'].message_type = _FEATURESET + _METHODOPTIONS.fields_by_name['uninterpreted_option'].message_type = _UNINTERPRETEDOPTION + _METHODOPTIONS_IDEMPOTENCYLEVEL.containing_type = _METHODOPTIONS + _UNINTERPRETEDOPTION_NAMEPART.containing_type = _UNINTERPRETEDOPTION + _UNINTERPRETEDOPTION.fields_by_name['name'].message_type = _UNINTERPRETEDOPTION_NAMEPART + _FEATURESET_VISIBILITYFEATURE.containing_type = _FEATURESET + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY.containing_type = _FEATURESET_VISIBILITYFEATURE + _FEATURESET.fields_by_name['field_presence'].enum_type = _FEATURESET_FIELDPRESENCE + _FEATURESET.fields_by_name['enum_type'].enum_type = _FEATURESET_ENUMTYPE + _FEATURESET.fields_by_name['repeated_field_encoding'].enum_type = _FEATURESET_REPEATEDFIELDENCODING + _FEATURESET.fields_by_name['utf8_validation'].enum_type = _FEATURESET_UTF8VALIDATION + _FEATURESET.fields_by_name['message_encoding'].enum_type = _FEATURESET_MESSAGEENCODING + _FEATURESET.fields_by_name['json_format'].enum_type = _FEATURESET_JSONFORMAT + _FEATURESET.fields_by_name['enforce_naming_style'].enum_type = _FEATURESET_ENFORCENAMINGSTYLE + _FEATURESET.fields_by_name['default_symbol_visibility'].enum_type = _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY + _FEATURESET_FIELDPRESENCE.containing_type = _FEATURESET + _FEATURESET_ENUMTYPE.containing_type = _FEATURESET + _FEATURESET_REPEATEDFIELDENCODING.containing_type = _FEATURESET + _FEATURESET_UTF8VALIDATION.containing_type = _FEATURESET + _FEATURESET_MESSAGEENCODING.containing_type = _FEATURESET + _FEATURESET_JSONFORMAT.containing_type = _FEATURESET + _FEATURESET_ENFORCENAMINGSTYLE.containing_type = _FEATURESET + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT.fields_by_name['edition'].enum_type = _EDITION + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT.fields_by_name['overridable_features'].message_type = _FEATURESET + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT.fields_by_name['fixed_features'].message_type = _FEATURESET + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT.containing_type = _FEATURESETDEFAULTS + _FEATURESETDEFAULTS.fields_by_name['defaults'].message_type = _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT + _FEATURESETDEFAULTS.fields_by_name['minimum_edition'].enum_type = _EDITION + _FEATURESETDEFAULTS.fields_by_name['maximum_edition'].enum_type = _EDITION + _SOURCECODEINFO_LOCATION.containing_type = _SOURCECODEINFO + _SOURCECODEINFO.fields_by_name['location'].message_type = _SOURCECODEINFO_LOCATION + _GENERATEDCODEINFO_ANNOTATION.fields_by_name['semantic'].enum_type = _GENERATEDCODEINFO_ANNOTATION_SEMANTIC + _GENERATEDCODEINFO_ANNOTATION.containing_type = _GENERATEDCODEINFO + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC.containing_type = _GENERATEDCODEINFO_ANNOTATION + _GENERATEDCODEINFO.fields_by_name['annotation'].message_type = _GENERATEDCODEINFO_ANNOTATION + DESCRIPTOR.message_types_by_name['FileDescriptorSet'] = _FILEDESCRIPTORSET + DESCRIPTOR.message_types_by_name['FileDescriptorProto'] = _FILEDESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['DescriptorProto'] = _DESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['ExtensionRangeOptions'] = _EXTENSIONRANGEOPTIONS + DESCRIPTOR.message_types_by_name['FieldDescriptorProto'] = _FIELDDESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['OneofDescriptorProto'] = _ONEOFDESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['EnumDescriptorProto'] = _ENUMDESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['EnumValueDescriptorProto'] = _ENUMVALUEDESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['ServiceDescriptorProto'] = _SERVICEDESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['MethodDescriptorProto'] = _METHODDESCRIPTORPROTO + DESCRIPTOR.message_types_by_name['FileOptions'] = _FILEOPTIONS + DESCRIPTOR.message_types_by_name['MessageOptions'] = _MESSAGEOPTIONS + DESCRIPTOR.message_types_by_name['FieldOptions'] = _FIELDOPTIONS + DESCRIPTOR.message_types_by_name['OneofOptions'] = _ONEOFOPTIONS + DESCRIPTOR.message_types_by_name['EnumOptions'] = _ENUMOPTIONS + DESCRIPTOR.message_types_by_name['EnumValueOptions'] = _ENUMVALUEOPTIONS + DESCRIPTOR.message_types_by_name['ServiceOptions'] = _SERVICEOPTIONS + DESCRIPTOR.message_types_by_name['MethodOptions'] = _METHODOPTIONS + DESCRIPTOR.message_types_by_name['UninterpretedOption'] = _UNINTERPRETEDOPTION + DESCRIPTOR.message_types_by_name['FeatureSet'] = _FEATURESET + DESCRIPTOR.message_types_by_name['FeatureSetDefaults'] = _FEATURESETDEFAULTS + DESCRIPTOR.message_types_by_name['SourceCodeInfo'] = _SOURCECODEINFO + DESCRIPTOR.message_types_by_name['GeneratedCodeInfo'] = _GENERATEDCODEINFO + DESCRIPTOR.enum_types_by_name['Edition'] = _EDITION + DESCRIPTOR.enum_types_by_name['SymbolVisibility'] = _SYMBOLVISIBILITY + _sym_db.RegisterFileDescriptor(DESCRIPTOR) + + class _ResolvedFeatures: + def __init__(self, features = None, **kwargs): + if features: + for k, v in features.FIELDS.items(): + setattr(self, k, getattr(features, k)) + else: + for k, v in kwargs.items(): + setattr(self, k, v) + DESCRIPTOR._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORSET._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORSET.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[10]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[11]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[12]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEDESCRIPTORPROTO.fields[13]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO.fields[10]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO_EXTENSIONRANGE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO_EXTENSIONRANGE.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO_EXTENSIONRANGE.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO_EXTENSIONRANGE.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO_RESERVEDRANGE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO_RESERVEDRANGE.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _DESCRIPTORPROTO_RESERVEDRANGE.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_DECLARATION._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_DECLARATION.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_DECLARATION.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_DECLARATION.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_DECLARATION.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_DECLARATION.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO.fields[10]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ONEOFDESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ONEOFDESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ONEOFDESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEDESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEDESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEDESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEDESCRIPTORPROTO.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEDESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEDESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEDESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEDESCRIPTORPROTO.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODDESCRIPTORPROTO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODDESCRIPTORPROTO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODDESCRIPTORPROTO.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODDESCRIPTORPROTO.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODDESCRIPTORPROTO.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODDESCRIPTORPROTO.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODDESCRIPTORPROTO.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[10]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[11]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[12]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[13]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[14]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[15]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[16]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[17]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[18]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[19]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS.fields[20]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _MESSAGEOPTIONS.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[10]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[11]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[12]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS.fields[13]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_EDITIONDEFAULT._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_EDITIONDEFAULT.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_EDITIONDEFAULT.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_FEATURESUPPORT._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_FEATURESUPPORT.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_FEATURESUPPORT.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_FEATURESUPPORT.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_FEATURESUPPORT.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ONEOFOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ONEOFOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ONEOFOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMOPTIONS.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMOPTIONS.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEOPTIONS.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _ENUMVALUEOPTIONS.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SERVICEOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION_NAMEPART._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION_NAMEPART.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["LEGACY_REQUIRED"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _UNINTERPRETEDOPTION_NAMEPART.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["LEGACY_REQUIRED"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET.fields[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_VISIBILITYFEATURE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO_LOCATION._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO_LOCATION.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["PACKED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO_LOCATION.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["PACKED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO_LOCATION.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO_LOCATION.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SOURCECODEINFO_LOCATION.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION.fields[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["PACKED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION.fields[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION.fields[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION.fields[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION.fields[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[10]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[11]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[12]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[13]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[14]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[15]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[16]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_TYPE.values[17]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_LABEL._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_LABEL.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_LABEL.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDDESCRIPTORPROTO_LABEL.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS_OPTIMIZEMODE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS_OPTIMIZEMODE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS_OPTIMIZEMODE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FILEOPTIONS_OPTIMIZEMODE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_CTYPE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_CTYPE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_CTYPE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_CTYPE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_JSTYPE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_JSTYPE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_JSTYPE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_JSTYPE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONRETENTION._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONRETENTION.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONRETENTION.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONRETENTION.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FIELDOPTIONS_OPTIONTARGETTYPE.values[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS_IDEMPOTENCYLEVEL._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS_IDEMPOTENCYLEVEL.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS_IDEMPOTENCYLEVEL.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _METHODOPTIONS_IDEMPOTENCYLEVEL.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_FIELDPRESENCE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_FIELDPRESENCE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_FIELDPRESENCE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_FIELDPRESENCE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_FIELDPRESENCE.values[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENUMTYPE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENUMTYPE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENUMTYPE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENUMTYPE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_REPEATEDFIELDENCODING._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_REPEATEDFIELDENCODING.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_REPEATEDFIELDENCODING.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_REPEATEDFIELDENCODING.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_UTF8VALIDATION._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_UTF8VALIDATION.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_UTF8VALIDATION.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_UTF8VALIDATION.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_MESSAGEENCODING._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_MESSAGEENCODING.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_MESSAGEENCODING.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_MESSAGEENCODING.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_JSONFORMAT._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_JSONFORMAT.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_JSONFORMAT.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_JSONFORMAT.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENFORCENAMINGSTYLE._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENFORCENAMINGSTYLE.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENFORCENAMINGSTYLE.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_ENFORCENAMINGSTYLE.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY.values[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY.values[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _GENERATEDCODEINFO_ANNOTATION_SEMANTIC.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[3]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[4]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[5]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[6]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[7]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[8]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[9]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[10]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[11]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _EDITION.values[12]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SYMBOLVISIBILITY._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SYMBOLVISIBILITY.values[0]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SYMBOLVISIBILITY.values[1]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) + _SYMBOLVISIBILITY.values[2]._features = _ResolvedFeatures(field_presence=_FEATURESET_FIELDPRESENCE.values_by_name["EXPLICIT"].number,enum_type=_FEATURESET_ENUMTYPE.values_by_name["CLOSED"].number,repeated_field_encoding=_FEATURESET_REPEATEDFIELDENCODING.values_by_name["EXPANDED"].number,utf8_validation=_FEATURESET_UTF8VALIDATION.values_by_name["NONE"].number,message_encoding=_FEATURESET_MESSAGEENCODING.values_by_name["LENGTH_PREFIXED"].number,json_format=_FEATURESET_JSONFORMAT.values_by_name["LEGACY_BEST_EFFORT"].number) +else: + _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.descriptor_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\020DescriptorProtosH\001Z-google.golang.org/protobuf/types/descriptorpb\370\001\001\242\002\003GPB\252\002\032Google.Protobuf.Reflection' + _globals['_EXTENSIONRANGEOPTIONS'].fields_by_name['declaration']._loaded_options = None + _globals['_EXTENSIONRANGEOPTIONS'].fields_by_name['declaration']._serialized_options = b'\210\001\002' + _globals['_EXTENSIONRANGEOPTIONS'].fields_by_name['verification']._loaded_options = None + _globals['_EXTENSIONRANGEOPTIONS'].fields_by_name['verification']._serialized_options = b'\210\001\002' + _globals['_FILEOPTIONS'].fields_by_name['java_generate_equals_and_hash']._loaded_options = None + _globals['_FILEOPTIONS'].fields_by_name['java_generate_equals_and_hash']._serialized_options = b'\030\001' + _globals['_MESSAGEOPTIONS'].fields_by_name['deprecated_legacy_json_field_conflicts']._loaded_options = None + _globals['_MESSAGEOPTIONS'].fields_by_name['deprecated_legacy_json_field_conflicts']._serialized_options = b'\030\001' + _globals['_FIELDOPTIONS'].fields_by_name['weak']._loaded_options = None + _globals['_FIELDOPTIONS'].fields_by_name['weak']._serialized_options = b'\030\001' + _globals['_ENUMOPTIONS'].fields_by_name['deprecated_legacy_json_field_conflicts']._loaded_options = None + _globals['_ENUMOPTIONS'].fields_by_name['deprecated_legacy_json_field_conflicts']._serialized_options = b'\030\001' + _globals['_FEATURESET'].fields_by_name['field_presence']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['field_presence']._serialized_options = b'\210\001\001\230\001\004\230\001\001\242\001\r\022\010EXPLICIT\030\204\007\242\001\r\022\010IMPLICIT\030\347\007\242\001\r\022\010EXPLICIT\030\350\007\262\001\003\010\350\007' + _globals['_FEATURESET'].fields_by_name['enum_type']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['enum_type']._serialized_options = b'\210\001\001\230\001\006\230\001\001\242\001\013\022\006CLOSED\030\204\007\242\001\t\022\004OPEN\030\347\007\262\001\003\010\350\007' + _globals['_FEATURESET'].fields_by_name['repeated_field_encoding']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['repeated_field_encoding']._serialized_options = b'\210\001\001\230\001\004\230\001\001\242\001\r\022\010EXPANDED\030\204\007\242\001\013\022\006PACKED\030\347\007\262\001\003\010\350\007' + _globals['_FEATURESET'].fields_by_name['utf8_validation']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['utf8_validation']._serialized_options = b'\210\001\001\230\001\004\230\001\001\242\001\t\022\004NONE\030\204\007\242\001\013\022\006VERIFY\030\347\007\262\001\003\010\350\007' + _globals['_FEATURESET'].fields_by_name['message_encoding']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['message_encoding']._serialized_options = b'\210\001\001\230\001\004\230\001\001\242\001\024\022\017LENGTH_PREFIXED\030\204\007\262\001\003\010\350\007' + _globals['_FEATURESET'].fields_by_name['json_format']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['json_format']._serialized_options = b'\210\001\001\230\001\003\230\001\006\230\001\001\242\001\027\022\022LEGACY_BEST_EFFORT\030\204\007\242\001\n\022\005ALLOW\030\347\007\262\001\003\010\350\007' + _globals['_FEATURESET'].fields_by_name['enforce_naming_style']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['enforce_naming_style']._serialized_options = b'\210\001\002\230\001\001\230\001\002\230\001\003\230\001\004\230\001\005\230\001\006\230\001\007\230\001\010\230\001\t\242\001\021\022\014STYLE_LEGACY\030\204\007\242\001\016\022\tSTYLE2024\030\351\007\262\001\003\010\351\007' + _globals['_FEATURESET'].fields_by_name['default_symbol_visibility']._loaded_options = None + _globals['_FEATURESET'].fields_by_name['default_symbol_visibility']._serialized_options = b'\210\001\002\230\001\001\242\001\017\022\nEXPORT_ALL\030\204\007\242\001\025\022\020EXPORT_TOP_LEVEL\030\351\007\262\001\003\010\351\007' + _globals['_SOURCECODEINFO_LOCATION'].fields_by_name['path']._loaded_options = None + _globals['_SOURCECODEINFO_LOCATION'].fields_by_name['path']._serialized_options = b'\020\001' + _globals['_SOURCECODEINFO_LOCATION'].fields_by_name['span']._loaded_options = None + _globals['_SOURCECODEINFO_LOCATION'].fields_by_name['span']._serialized_options = b'\020\001' + _globals['_GENERATEDCODEINFO_ANNOTATION'].fields_by_name['path']._loaded_options = None + _globals['_GENERATEDCODEINFO_ANNOTATION'].fields_by_name['path']._serialized_options = b'\020\001' + _globals['_EDITION']._serialized_start=12685 + _globals['_EDITION']._serialized_end=13003 + _globals['_SYMBOLVISIBILITY']._serialized_start=13005 + _globals['_SYMBOLVISIBILITY']._serialized_end=13090 + _globals['_FILEDESCRIPTORSET']._serialized_start=53 + _globals['_FILEDESCRIPTORSET']._serialized_end=144 + _globals['_FILEDESCRIPTORPROTO']._serialized_start=147 + _globals['_FILEDESCRIPTORPROTO']._serialized_end=856 + _globals['_DESCRIPTORPROTO']._serialized_start=859 + _globals['_DESCRIPTORPROTO']._serialized_end=1751 + _globals['_DESCRIPTORPROTO_EXTENSIONRANGE']._serialized_start=1572 + _globals['_DESCRIPTORPROTO_EXTENSIONRANGE']._serialized_end=1694 + _globals['_DESCRIPTORPROTO_RESERVEDRANGE']._serialized_start=1696 + _globals['_DESCRIPTORPROTO_RESERVEDRANGE']._serialized_end=1751 + _globals['_EXTENSIONRANGEOPTIONS']._serialized_start=1754 + _globals['_EXTENSIONRANGEOPTIONS']._serialized_end=2342 + _globals['_EXTENSIONRANGEOPTIONS_DECLARATION']._serialized_start=2129 + _globals['_EXTENSIONRANGEOPTIONS_DECLARATION']._serialized_end=2277 + _globals['_EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE']._serialized_start=2279 + _globals['_EXTENSIONRANGEOPTIONS_VERIFICATIONSTATE']._serialized_end=2331 + _globals['_FIELDDESCRIPTORPROTO']._serialized_start=2345 + _globals['_FIELDDESCRIPTORPROTO']._serialized_end=3178 + _globals['_FIELDDESCRIPTORPROTO_TYPE']._serialized_start=2799 + _globals['_FIELDDESCRIPTORPROTO_TYPE']._serialized_end=3109 + _globals['_FIELDDESCRIPTORPROTO_LABEL']._serialized_start=3111 + _globals['_FIELDDESCRIPTORPROTO_LABEL']._serialized_end=3178 + _globals['_ONEOFDESCRIPTORPROTO']._serialized_start=3180 + _globals['_ONEOFDESCRIPTORPROTO']._serialized_end=3279 + _globals['_ENUMDESCRIPTORPROTO']._serialized_start=3282 + _globals['_ENUMDESCRIPTORPROTO']._serialized_end=3704 + _globals['_ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE']._serialized_start=3645 + _globals['_ENUMDESCRIPTORPROTO_ENUMRESERVEDRANGE']._serialized_end=3704 + _globals['_ENUMVALUEDESCRIPTORPROTO']._serialized_start=3707 + _globals['_ENUMVALUEDESCRIPTORPROTO']._serialized_end=3838 + _globals['_SERVICEDESCRIPTORPROTO']._serialized_start=3841 + _globals['_SERVICEDESCRIPTORPROTO']._serialized_end=4022 + _globals['_METHODDESCRIPTORPROTO']._serialized_start=4025 + _globals['_METHODDESCRIPTORPROTO']._serialized_end=4290 + _globals['_FILEOPTIONS']._serialized_start=4293 + _globals['_FILEOPTIONS']._serialized_end=5490 + _globals['_FILEOPTIONS_OPTIMIZEMODE']._serialized_start=5387 + _globals['_FILEOPTIONS_OPTIMIZEMODE']._serialized_end=5445 + _globals['_MESSAGEOPTIONS']._serialized_start=5493 + _globals['_MESSAGEOPTIONS']._serialized_end=5993 + _globals['_FIELDOPTIONS']._serialized_start=5996 + _globals['_FIELDOPTIONS']._serialized_end=7693 + _globals['_FIELDOPTIONS_EDITIONDEFAULT']._serialized_start=6837 + _globals['_FIELDOPTIONS_EDITIONDEFAULT']._serialized_end=6927 + _globals['_FIELDOPTIONS_FEATURESUPPORT']._serialized_start=6930 + _globals['_FIELDOPTIONS_FEATURESUPPORT']._serialized_end=7208 + _globals['_FIELDOPTIONS_CTYPE']._serialized_start=7210 + _globals['_FIELDOPTIONS_CTYPE']._serialized_end=7257 + _globals['_FIELDOPTIONS_JSTYPE']._serialized_start=7259 + _globals['_FIELDOPTIONS_JSTYPE']._serialized_end=7312 + _globals['_FIELDOPTIONS_OPTIONRETENTION']._serialized_start=7314 + _globals['_FIELDOPTIONS_OPTIONRETENTION']._serialized_end=7399 + _globals['_FIELDOPTIONS_OPTIONTARGETTYPE']._serialized_start=7402 + _globals['_FIELDOPTIONS_OPTIONTARGETTYPE']._serialized_end=7670 + _globals['_ONEOFOPTIONS']._serialized_start=7696 + _globals['_ONEOFOPTIONS']._serialized_end=7868 + _globals['_ENUMOPTIONS']._serialized_start=7871 + _globals['_ENUMOPTIONS']._serialized_end=8208 + _globals['_ENUMVALUEOPTIONS']._serialized_start=8211 + _globals['_ENUMVALUEOPTIONS']._serialized_end=8555 + _globals['_SERVICEOPTIONS']._serialized_start=8558 + _globals['_SERVICEOPTIONS']._serialized_end=8771 + _globals['_METHODOPTIONS']._serialized_start=8774 + _globals['_METHODOPTIONS']._serialized_end=9183 + _globals['_METHODOPTIONS_IDEMPOTENCYLEVEL']._serialized_start=9092 + _globals['_METHODOPTIONS_IDEMPOTENCYLEVEL']._serialized_end=9172 + _globals['_UNINTERPRETEDOPTION']._serialized_start=9186 + _globals['_UNINTERPRETEDOPTION']._serialized_end=9596 + _globals['_UNINTERPRETEDOPTION_NAMEPART']._serialized_start=9522 + _globals['_UNINTERPRETEDOPTION_NAMEPART']._serialized_end=9596 + _globals['_FEATURESET']._serialized_start=9599 + _globals['_FEATURESET']._serialized_end=11533 + _globals['_FEATURESET_VISIBILITYFEATURE']._serialized_start=10778 + _globals['_FEATURESET_VISIBILITYFEATURE']._serialized_end=10939 + _globals['_FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY']._serialized_start=10800 + _globals['_FEATURESET_VISIBILITYFEATURE_DEFAULTSYMBOLVISIBILITY']._serialized_end=10929 + _globals['_FEATURESET_FIELDPRESENCE']._serialized_start=10941 + _globals['_FEATURESET_FIELDPRESENCE']._serialized_end=11033 + _globals['_FEATURESET_ENUMTYPE']._serialized_start=11035 + _globals['_FEATURESET_ENUMTYPE']._serialized_end=11090 + _globals['_FEATURESET_REPEATEDFIELDENCODING']._serialized_start=11092 + _globals['_FEATURESET_REPEATEDFIELDENCODING']._serialized_end=11178 + _globals['_FEATURESET_UTF8VALIDATION']._serialized_start=11180 + _globals['_FEATURESET_UTF8VALIDATION']._serialized_end=11253 + _globals['_FEATURESET_MESSAGEENCODING']._serialized_start=11255 + _globals['_FEATURESET_MESSAGEENCODING']._serialized_end=11338 + _globals['_FEATURESET_JSONFORMAT']._serialized_start=11340 + _globals['_FEATURESET_JSONFORMAT']._serialized_end=11412 + _globals['_FEATURESET_ENFORCENAMINGSTYLE']._serialized_start=11414 + _globals['_FEATURESET_ENFORCENAMINGSTYLE']._serialized_end=11501 + _globals['_FEATURESETDEFAULTS']._serialized_start=11536 + _globals['_FEATURESETDEFAULTS']._serialized_end=12031 + _globals['_FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT']._serialized_start=11783 + _globals['_FEATURESETDEFAULTS_FEATURESETEDITIONDEFAULT']._serialized_end=12031 + _globals['_SOURCECODEINFO']._serialized_start=12034 + _globals['_SOURCECODEINFO']._serialized_end=12343 + _globals['_SOURCECODEINFO_LOCATION']._serialized_start=12123 + _globals['_SOURCECODEINFO_LOCATION']._serialized_end=12329 + _globals['_GENERATEDCODEINFO']._serialized_start=12346 + _globals['_GENERATEDCODEINFO']._serialized_end=12682 + _globals['_GENERATEDCODEINFO_ANNOTATION']._serialized_start=12447 + _globals['_GENERATEDCODEINFO_ANNOTATION']._serialized_end=12682 + _globals['_GENERATEDCODEINFO_ANNOTATION_SEMANTIC']._serialized_start=12642 + _globals['_GENERATEDCODEINFO_ANNOTATION_SEMANTIC']._serialized_end=12682 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/descriptor_pool.py b/lib/python3.12/site-packages/google/protobuf/descriptor_pool.py new file mode 100644 index 0000000000000000000000000000000000000000..a838c508b58b2f3e06fb7cbf80e24ec2e464e215 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/descriptor_pool.py @@ -0,0 +1,1373 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Provides DescriptorPool to use as a container for proto2 descriptors. + +The DescriptorPool is used in conjection with a DescriptorDatabase to maintain +a collection of protocol buffer descriptors for use when dynamically creating +message types at runtime. + +For most applications protocol buffers should be used via modules generated by +the protocol buffer compiler tool. This should only be used when the type of +protocol buffers used in an application or library cannot be predetermined. + +Below is a straightforward example on how to use this class:: + + pool = DescriptorPool() + file_descriptor_protos = [ ... ] + for file_descriptor_proto in file_descriptor_protos: + pool.Add(file_descriptor_proto) + my_message_descriptor = pool.FindMessageTypeByName('some.package.MessageType') + +The message descriptor can be used in conjunction with the message_factory +module in order to create a protocol buffer class that can be encoded and +decoded. + +If you want to get a Python class for the specified proto, use the +helper functions inside google.protobuf.message_factory +directly instead of this class. +""" + +__author__ = 'matthewtoia@google.com (Matt Toia)' + +import collections +import threading +import warnings + +from google.protobuf import descriptor +from google.protobuf import descriptor_database +from google.protobuf import text_encoding +from google.protobuf.internal import python_edition_defaults +from google.protobuf.internal import python_message + +_USE_C_DESCRIPTORS = descriptor._USE_C_DESCRIPTORS # pylint: disable=protected-access + + +def _NormalizeFullyQualifiedName(name): + """Remove leading period from fully-qualified type name. + + Due to b/13860351 in descriptor_database.py, types in the root namespace are + generated with a leading period. This function removes that prefix. + + Args: + name (str): The fully-qualified symbol name. + + Returns: + str: The normalized fully-qualified symbol name. + """ + return name.lstrip('.') + + +def _OptionsOrNone(descriptor_proto): + """Returns the value of the field `options`, or None if it is not set.""" + if descriptor_proto.HasField('options'): + return descriptor_proto.options + else: + return None + + +def _IsMessageSetExtension(field): + return (field.is_extension and + field.containing_type.has_options and + field.containing_type.GetOptions().message_set_wire_format and + field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + not field.is_required and + not field.is_repeated) + +_edition_defaults_lock = threading.Lock() + + +class DescriptorPool(object): + """A collection of protobufs dynamically constructed by descriptor protos.""" + + if _USE_C_DESCRIPTORS: + + def __new__(cls, descriptor_db=None): + # pylint: disable=protected-access + return descriptor._message.DescriptorPool(descriptor_db) + + def __init__( + self, descriptor_db=None, use_deprecated_legacy_json_field_conflicts=False + ): + """Initializes a Pool of proto buffs. + + The descriptor_db argument to the constructor is provided to allow + specialized file descriptor proto lookup code to be triggered on demand. An + example would be an implementation which will read and compile a file + specified in a call to FindFileByName() and not require the call to Add() + at all. Results from this database will be cached internally here as well. + + Args: + descriptor_db: A secondary source of file descriptors. + use_deprecated_legacy_json_field_conflicts: Unused, for compatibility with + C++. + """ + + self._internal_db = descriptor_database.DescriptorDatabase() + self._descriptor_db = descriptor_db + self._descriptors = {} + self._enum_descriptors = {} + self._service_descriptors = {} + self._file_descriptors = {} + self._toplevel_extensions = {} + self._top_enum_values = {} + # We store extensions in two two-level mappings: The first key is the + # descriptor of the message being extended, the second key is the extension + # full name or its tag number. + self._extensions_by_name = collections.defaultdict(dict) + self._extensions_by_number = collections.defaultdict(dict) + self._serialized_edition_defaults = ( + python_edition_defaults._PROTOBUF_INTERNAL_PYTHON_EDITION_DEFAULTS + ) + self._edition_defaults = None + self._feature_cache = dict() + + def _CheckConflictRegister(self, desc, desc_name, file_name): + """Check if the descriptor name conflicts with another of the same name. + + Args: + desc: Descriptor of a message, enum, service, extension or enum value. + desc_name (str): the full name of desc. + file_name (str): The file name of descriptor. + """ + for register, descriptor_type in [ + (self._descriptors, descriptor.Descriptor), + (self._enum_descriptors, descriptor.EnumDescriptor), + (self._service_descriptors, descriptor.ServiceDescriptor), + (self._toplevel_extensions, descriptor.FieldDescriptor), + (self._top_enum_values, descriptor.EnumValueDescriptor)]: + if desc_name in register: + old_desc = register[desc_name] + if isinstance(old_desc, descriptor.EnumValueDescriptor): + old_file = old_desc.type.file.name + else: + old_file = old_desc.file.name + + if not isinstance(desc, descriptor_type) or ( + old_file != file_name): + error_msg = ('Conflict register for file "' + file_name + + '": ' + desc_name + + ' is already defined in file "' + + old_file + '". Please fix the conflict by adding ' + 'package name on the proto file, or use different ' + 'name for the duplication.') + if isinstance(desc, descriptor.EnumValueDescriptor): + error_msg += ('\nNote: enum values appear as ' + 'siblings of the enum type instead of ' + 'children of it.') + + raise TypeError(error_msg) + + return + + def Add(self, file_desc_proto): + """Adds the FileDescriptorProto and its types to this pool. + + Args: + file_desc_proto (FileDescriptorProto): The file descriptor to add. + """ + + self._internal_db.Add(file_desc_proto) + + def AddSerializedFile(self, serialized_file_desc_proto): + """Adds the FileDescriptorProto and its types to this pool. + + Args: + serialized_file_desc_proto (bytes): A bytes string, serialization of the + :class:`FileDescriptorProto` to add. + + Returns: + FileDescriptor: Descriptor for the added file. + """ + + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + file_desc_proto = descriptor_pb2.FileDescriptorProto.FromString( + serialized_file_desc_proto) + file_desc = self._ConvertFileProtoToFileDescriptor(file_desc_proto) + file_desc.serialized_pb = serialized_file_desc_proto + return file_desc + + # Never call this method. It is for internal usage only. + def _AddDescriptor(self, desc): + """Adds a Descriptor to the pool, non-recursively. + + If the Descriptor contains nested messages or enums, the caller must + explicitly register them. This method also registers the FileDescriptor + associated with the message. + + Args: + desc: A Descriptor. + """ + if not isinstance(desc, descriptor.Descriptor): + raise TypeError('Expected instance of descriptor.Descriptor.') + + self._CheckConflictRegister(desc, desc.full_name, desc.file.name) + + self._descriptors[desc.full_name] = desc + self._AddFileDescriptor(desc.file) + + # Never call this method. It is for internal usage only. + def _AddEnumDescriptor(self, enum_desc): + """Adds an EnumDescriptor to the pool. + + This method also registers the FileDescriptor associated with the enum. + + Args: + enum_desc: An EnumDescriptor. + """ + + if not isinstance(enum_desc, descriptor.EnumDescriptor): + raise TypeError('Expected instance of descriptor.EnumDescriptor.') + + file_name = enum_desc.file.name + self._CheckConflictRegister(enum_desc, enum_desc.full_name, file_name) + self._enum_descriptors[enum_desc.full_name] = enum_desc + + # Top enum values need to be indexed. + # Count the number of dots to see whether the enum is toplevel or nested + # in a message. We cannot use enum_desc.containing_type at this stage. + if enum_desc.file.package: + top_level = (enum_desc.full_name.count('.') + - enum_desc.file.package.count('.') == 1) + else: + top_level = enum_desc.full_name.count('.') == 0 + if top_level: + file_name = enum_desc.file.name + package = enum_desc.file.package + for enum_value in enum_desc.values: + full_name = _NormalizeFullyQualifiedName( + '.'.join((package, enum_value.name))) + self._CheckConflictRegister(enum_value, full_name, file_name) + self._top_enum_values[full_name] = enum_value + self._AddFileDescriptor(enum_desc.file) + + # Never call this method. It is for internal usage only. + def _AddServiceDescriptor(self, service_desc): + """Adds a ServiceDescriptor to the pool. + + Args: + service_desc: A ServiceDescriptor. + """ + + if not isinstance(service_desc, descriptor.ServiceDescriptor): + raise TypeError('Expected instance of descriptor.ServiceDescriptor.') + + self._CheckConflictRegister(service_desc, service_desc.full_name, + service_desc.file.name) + self._service_descriptors[service_desc.full_name] = service_desc + + # Never call this method. It is for internal usage only. + def _AddExtensionDescriptor(self, extension): + """Adds a FieldDescriptor describing an extension to the pool. + + Args: + extension: A FieldDescriptor. + + Raises: + AssertionError: when another extension with the same number extends the + same message. + TypeError: when the specified extension is not a + descriptor.FieldDescriptor. + """ + if not (isinstance(extension, descriptor.FieldDescriptor) and + extension.is_extension): + raise TypeError('Expected an extension descriptor.') + + if extension.extension_scope is None: + self._CheckConflictRegister( + extension, extension.full_name, extension.file.name) + self._toplevel_extensions[extension.full_name] = extension + + try: + existing_desc = self._extensions_by_number[ + extension.containing_type][extension.number] + except KeyError: + pass + else: + if extension is not existing_desc: + raise AssertionError( + 'Extensions "%s" and "%s" both try to extend message type "%s" ' + 'with field number %d.' % + (extension.full_name, existing_desc.full_name, + extension.containing_type.full_name, extension.number)) + + self._extensions_by_number[extension.containing_type][ + extension.number] = extension + self._extensions_by_name[extension.containing_type][ + extension.full_name] = extension + + # Also register MessageSet extensions with the type name. + if _IsMessageSetExtension(extension): + self._extensions_by_name[extension.containing_type][ + extension.message_type.full_name] = extension + + if hasattr(extension.containing_type, '_concrete_class'): + python_message._AttachFieldHelpers( + extension.containing_type._concrete_class, extension) + + # Never call this method. It is for internal usage only. + def _InternalAddFileDescriptor(self, file_desc): + """Adds a FileDescriptor to the pool, non-recursively. + + If the FileDescriptor contains messages or enums, the caller must explicitly + register them. + + Args: + file_desc: A FileDescriptor. + """ + + self._AddFileDescriptor(file_desc) + + def _AddFileDescriptor(self, file_desc): + """Adds a FileDescriptor to the pool, non-recursively. + + If the FileDescriptor contains messages or enums, the caller must explicitly + register them. + + Args: + file_desc: A FileDescriptor. + """ + + if not isinstance(file_desc, descriptor.FileDescriptor): + raise TypeError('Expected instance of descriptor.FileDescriptor.') + self._file_descriptors[file_desc.name] = file_desc + + def FindFileByName(self, file_name): + """Gets a FileDescriptor by file name. + + Args: + file_name (str): The path to the file to get a descriptor for. + + Returns: + FileDescriptor: The descriptor for the named file. + + Raises: + KeyError: if the file cannot be found in the pool. + """ + + try: + return self._file_descriptors[file_name] + except KeyError: + pass + + try: + file_proto = self._internal_db.FindFileByName(file_name) + except KeyError as error: + if self._descriptor_db: + file_proto = self._descriptor_db.FindFileByName(file_name) + else: + raise error + if not file_proto: + raise KeyError('Cannot find a file named %s' % file_name) + return self._ConvertFileProtoToFileDescriptor(file_proto) + + def FindFileContainingSymbol(self, symbol): + """Gets the FileDescriptor for the file containing the specified symbol. + + Args: + symbol (str): The name of the symbol to search for. + + Returns: + FileDescriptor: Descriptor for the file that contains the specified + symbol. + + Raises: + KeyError: if the file cannot be found in the pool. + """ + + symbol = _NormalizeFullyQualifiedName(symbol) + try: + return self._InternalFindFileContainingSymbol(symbol) + except KeyError: + pass + + try: + # Try fallback database. Build and find again if possible. + self._FindFileContainingSymbolInDb(symbol) + return self._InternalFindFileContainingSymbol(symbol) + except KeyError: + raise KeyError('Cannot find a file containing %s' % symbol) + + def _InternalFindFileContainingSymbol(self, symbol): + """Gets the already built FileDescriptor containing the specified symbol. + + Args: + symbol (str): The name of the symbol to search for. + + Returns: + FileDescriptor: Descriptor for the file that contains the specified + symbol. + + Raises: + KeyError: if the file cannot be found in the pool. + """ + try: + return self._descriptors[symbol].file + except KeyError: + pass + + try: + return self._enum_descriptors[symbol].file + except KeyError: + pass + + try: + return self._service_descriptors[symbol].file + except KeyError: + pass + + try: + return self._top_enum_values[symbol].type.file + except KeyError: + pass + + try: + return self._toplevel_extensions[symbol].file + except KeyError: + pass + + # Try fields, enum values and nested extensions inside a message. + top_name, _, sub_name = symbol.rpartition('.') + try: + message = self.FindMessageTypeByName(top_name) + assert (sub_name in message.extensions_by_name or + sub_name in message.fields_by_name or + sub_name in message.enum_values_by_name) + return message.file + except (KeyError, AssertionError): + raise KeyError('Cannot find a file containing %s' % symbol) + + def FindMessageTypeByName(self, full_name): + """Loads the named descriptor from the pool. + + Args: + full_name (str): The full name of the descriptor to load. + + Returns: + Descriptor: The descriptor for the named type. + + Raises: + KeyError: if the message cannot be found in the pool. + """ + + full_name = _NormalizeFullyQualifiedName(full_name) + if full_name not in self._descriptors: + self._FindFileContainingSymbolInDb(full_name) + return self._descriptors[full_name] + + def FindEnumTypeByName(self, full_name): + """Loads the named enum descriptor from the pool. + + Args: + full_name (str): The full name of the enum descriptor to load. + + Returns: + EnumDescriptor: The enum descriptor for the named type. + + Raises: + KeyError: if the enum cannot be found in the pool. + """ + + full_name = _NormalizeFullyQualifiedName(full_name) + if full_name not in self._enum_descriptors: + self._FindFileContainingSymbolInDb(full_name) + return self._enum_descriptors[full_name] + + def FindFieldByName(self, full_name): + """Loads the named field descriptor from the pool. + + Args: + full_name (str): The full name of the field descriptor to load. + + Returns: + FieldDescriptor: The field descriptor for the named field. + + Raises: + KeyError: if the field cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + message_name, _, field_name = full_name.rpartition('.') + message_descriptor = self.FindMessageTypeByName(message_name) + return message_descriptor.fields_by_name[field_name] + + def FindOneofByName(self, full_name): + """Loads the named oneof descriptor from the pool. + + Args: + full_name (str): The full name of the oneof descriptor to load. + + Returns: + OneofDescriptor: The oneof descriptor for the named oneof. + + Raises: + KeyError: if the oneof cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + message_name, _, oneof_name = full_name.rpartition('.') + message_descriptor = self.FindMessageTypeByName(message_name) + return message_descriptor.oneofs_by_name[oneof_name] + + def FindExtensionByName(self, full_name): + """Loads the named extension descriptor from the pool. + + Args: + full_name (str): The full name of the extension descriptor to load. + + Returns: + FieldDescriptor: The field descriptor for the named extension. + + Raises: + KeyError: if the extension cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + try: + # The proto compiler does not give any link between the FileDescriptor + # and top-level extensions unless the FileDescriptorProto is added to + # the DescriptorDatabase, but this can impact memory usage. + # So we registered these extensions by name explicitly. + return self._toplevel_extensions[full_name] + except KeyError: + pass + message_name, _, extension_name = full_name.rpartition('.') + try: + # Most extensions are nested inside a message. + scope = self.FindMessageTypeByName(message_name) + except KeyError: + # Some extensions are defined at file scope. + scope = self._FindFileContainingSymbolInDb(full_name) + return scope.extensions_by_name[extension_name] + + def FindExtensionByNumber(self, message_descriptor, number): + """Gets the extension of the specified message with the specified number. + + Extensions have to be registered to this pool by calling :func:`Add` or + :func:`AddExtensionDescriptor`. + + Args: + message_descriptor (Descriptor): descriptor of the extended message. + number (int): Number of the extension field. + + Returns: + FieldDescriptor: The descriptor for the extension. + + Raises: + KeyError: when no extension with the given number is known for the + specified message. + """ + try: + return self._extensions_by_number[message_descriptor][number] + except KeyError: + self._TryLoadExtensionFromDB(message_descriptor, number) + return self._extensions_by_number[message_descriptor][number] + + def FindAllExtensions(self, message_descriptor): + """Gets all the known extensions of a given message. + + Extensions have to be registered to this pool by build related + :func:`Add` or :func:`AddExtensionDescriptor`. + + Args: + message_descriptor (Descriptor): Descriptor of the extended message. + + Returns: + list[FieldDescriptor]: Field descriptors describing the extensions. + """ + # Fallback to descriptor db if FindAllExtensionNumbers is provided. + if self._descriptor_db and hasattr( + self._descriptor_db, 'FindAllExtensionNumbers'): + full_name = message_descriptor.full_name + try: + all_numbers = self._descriptor_db.FindAllExtensionNumbers(full_name) + except: + pass + else: + if isinstance(all_numbers, list): + for number in all_numbers: + if number in self._extensions_by_number[message_descriptor]: + continue + self._TryLoadExtensionFromDB(message_descriptor, number) + else: + warnings.warn( + 'FindAllExtensionNumbers() on fall back DB must return a list,' + ' not {0}'.format(type(all_numbers)) + ) + + return list(self._extensions_by_number[message_descriptor].values()) + + def _TryLoadExtensionFromDB(self, message_descriptor, number): + """Try to Load extensions from descriptor db. + + Args: + message_descriptor: descriptor of the extended message. + number: the extension number that needs to be loaded. + """ + if not self._descriptor_db: + return + # Only supported when FindFileContainingExtension is provided. + if not hasattr( + self._descriptor_db, 'FindFileContainingExtension'): + return + + full_name = message_descriptor.full_name + file_proto = None + try: + file_proto = self._descriptor_db.FindFileContainingExtension( + full_name, number + ) + except: + return + + if file_proto is None: + return + + try: + self._ConvertFileProtoToFileDescriptor(file_proto) + except: + warn_msg = ('Unable to load proto file %s for extension number %d.' % + (file_proto.name, number)) + warnings.warn(warn_msg, RuntimeWarning) + + def FindServiceByName(self, full_name): + """Loads the named service descriptor from the pool. + + Args: + full_name (str): The full name of the service descriptor to load. + + Returns: + ServiceDescriptor: The service descriptor for the named service. + + Raises: + KeyError: if the service cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + if full_name not in self._service_descriptors: + self._FindFileContainingSymbolInDb(full_name) + return self._service_descriptors[full_name] + + def FindMethodByName(self, full_name): + """Loads the named service method descriptor from the pool. + + Args: + full_name (str): The full name of the method descriptor to load. + + Returns: + MethodDescriptor: The method descriptor for the service method. + + Raises: + KeyError: if the method cannot be found in the pool. + """ + full_name = _NormalizeFullyQualifiedName(full_name) + service_name, _, method_name = full_name.rpartition('.') + service_descriptor = self.FindServiceByName(service_name) + return service_descriptor.methods_by_name[method_name] + + def SetFeatureSetDefaults(self, defaults): + """Sets the default feature mappings used during the build. + + Args: + defaults: a FeatureSetDefaults message containing the new mappings. + """ + if self._edition_defaults is not None: + raise ValueError( + "Feature set defaults can't be changed once the pool has started" + ' building!' + ) + + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + + if not isinstance(defaults, descriptor_pb2.FeatureSetDefaults): + raise TypeError('SetFeatureSetDefaults called with invalid type') + + if defaults.minimum_edition > defaults.maximum_edition: + raise ValueError( + 'Invalid edition range %s to %s' + % ( + descriptor_pb2.Edition.Name(defaults.minimum_edition), + descriptor_pb2.Edition.Name(defaults.maximum_edition), + ) + ) + + prev_edition = descriptor_pb2.Edition.EDITION_UNKNOWN + for d in defaults.defaults: + if d.edition == descriptor_pb2.Edition.EDITION_UNKNOWN: + raise ValueError('Invalid edition EDITION_UNKNOWN specified') + if prev_edition >= d.edition: + raise ValueError( + 'Feature set defaults are not strictly increasing. %s is greater' + ' than or equal to %s' + % ( + descriptor_pb2.Edition.Name(prev_edition), + descriptor_pb2.Edition.Name(d.edition), + ) + ) + prev_edition = d.edition + self._edition_defaults = defaults + + def _CreateDefaultFeatures(self, edition): + """Creates a FeatureSet message with defaults for a specific edition. + + Args: + edition: the edition to generate defaults for. + + Returns: + A FeatureSet message with defaults for a specific edition. + """ + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + + with _edition_defaults_lock: + if not self._edition_defaults: + self._edition_defaults = descriptor_pb2.FeatureSetDefaults() + self._edition_defaults.ParseFromString( + self._serialized_edition_defaults + ) + + if edition < self._edition_defaults.minimum_edition: + raise TypeError( + 'Edition %s is earlier than the minimum supported edition %s!' + % ( + descriptor_pb2.Edition.Name(edition), + descriptor_pb2.Edition.Name( + self._edition_defaults.minimum_edition + ), + ) + ) + if ( + edition > self._edition_defaults.maximum_edition + and edition != descriptor_pb2.EDITION_UNSTABLE + ): + raise TypeError( + 'Edition %s is later than the maximum supported edition %s!' + % ( + descriptor_pb2.Edition.Name(edition), + descriptor_pb2.Edition.Name( + self._edition_defaults.maximum_edition + ), + ) + ) + found = None + for d in self._edition_defaults.defaults: + if d.edition > edition: + break + found = d + if found is None: + raise TypeError( + 'No valid default found for edition %s!' + % descriptor_pb2.Edition.Name(edition) + ) + + defaults = descriptor_pb2.FeatureSet() + defaults.CopyFrom(found.fixed_features) + defaults.MergeFrom(found.overridable_features) + return defaults + + def _InternFeatures(self, features): + serialized = features.SerializeToString() + with _edition_defaults_lock: + cached = self._feature_cache.get(serialized) + if cached is None: + self._feature_cache[serialized] = features + cached = features + return cached + + def _FindFileContainingSymbolInDb(self, symbol): + """Finds the file in descriptor DB containing the specified symbol. + + Args: + symbol (str): The name of the symbol to search for. + + Returns: + FileDescriptor: The file that contains the specified symbol. + + Raises: + KeyError: if the file cannot be found in the descriptor database. + """ + try: + file_proto = self._internal_db.FindFileContainingSymbol(symbol) + except KeyError as error: + if self._descriptor_db: + file_proto = self._descriptor_db.FindFileContainingSymbol(symbol) + else: + raise error + if not file_proto: + raise KeyError('Cannot find a file containing %s' % symbol) + return self._ConvertFileProtoToFileDescriptor(file_proto) + + def _ConvertFileProtoToFileDescriptor(self, file_proto): + """Creates a FileDescriptor from a proto or returns a cached copy. + + This method also has the side effect of loading all the symbols found in + the file into the appropriate dictionaries in the pool. + + Args: + file_proto: The proto to convert. + + Returns: + A FileDescriptor matching the passed in proto. + """ + if file_proto.name not in self._file_descriptors: + built_deps = list(self._GetDeps(file_proto.dependency)) + direct_deps = [self.FindFileByName(n) for n in file_proto.dependency] + public_deps = [direct_deps[i] for i in file_proto.public_dependency] + + # pylint: disable=g-import-not-at-top + from google.protobuf import descriptor_pb2 + + file_descriptor = descriptor.FileDescriptor( + pool=self, + name=file_proto.name, + package=file_proto.package, + syntax=file_proto.syntax, + edition=descriptor_pb2.Edition.Name(file_proto.edition), + options=_OptionsOrNone(file_proto), + serialized_pb=file_proto.SerializeToString(), + dependencies=direct_deps, + public_dependencies=public_deps, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key, + ) + scope = {} + + # This loop extracts all the message and enum types from all the + # dependencies of the file_proto. This is necessary to create the + # scope of available message types when defining the passed in + # file proto. + for dependency in built_deps: + scope.update(self._ExtractSymbols( + dependency.message_types_by_name.values())) + scope.update((_PrefixWithDot(enum.full_name), enum) + for enum in dependency.enum_types_by_name.values()) + + for message_type in file_proto.message_type: + message_desc = self._ConvertMessageDescriptor( + message_type, file_proto.package, file_descriptor, scope, + file_proto.syntax) + file_descriptor.message_types_by_name[message_desc.name] = ( + message_desc) + + for enum_type in file_proto.enum_type: + file_descriptor.enum_types_by_name[enum_type.name] = ( + self._ConvertEnumDescriptor(enum_type, file_proto.package, + file_descriptor, None, scope, True)) + + for index, extension_proto in enumerate(file_proto.extension): + extension_desc = self._MakeFieldDescriptor( + extension_proto, file_proto.package, index, file_descriptor, + is_extension=True) + extension_desc.containing_type = self._GetTypeFromScope( + file_descriptor.package, extension_proto.extendee, scope) + self._SetFieldType(extension_proto, extension_desc, + file_descriptor.package, scope) + file_descriptor.extensions_by_name[extension_desc.name] = ( + extension_desc) + + for desc_proto in file_proto.message_type: + self._SetAllFieldTypes(file_proto.package, desc_proto, scope) + + if file_proto.package: + desc_proto_prefix = _PrefixWithDot(file_proto.package) + else: + desc_proto_prefix = '' + + for desc_proto in file_proto.message_type: + desc = self._GetTypeFromScope( + desc_proto_prefix, desc_proto.name, scope) + file_descriptor.message_types_by_name[desc_proto.name] = desc + + for index, service_proto in enumerate(file_proto.service): + file_descriptor.services_by_name[service_proto.name] = ( + self._MakeServiceDescriptor(service_proto, index, scope, + file_proto.package, file_descriptor)) + + self._file_descriptors[file_proto.name] = file_descriptor + + # Add extensions to the pool + def AddExtensionForNested(message_type): + for nested in message_type.nested_types: + AddExtensionForNested(nested) + for extension in message_type.extensions: + self._AddExtensionDescriptor(extension) + + file_desc = self._file_descriptors[file_proto.name] + for extension in file_desc.extensions_by_name.values(): + self._AddExtensionDescriptor(extension) + for message_type in file_desc.message_types_by_name.values(): + AddExtensionForNested(message_type) + + return file_desc + + def _ConvertMessageDescriptor(self, desc_proto, package=None, file_desc=None, + scope=None, syntax=None): + """Adds the proto to the pool in the specified package. + + Args: + desc_proto: The descriptor_pb2.DescriptorProto protobuf message. + package: The package the proto should be located in. + file_desc: The file containing this message. + scope: Dict mapping short and full symbols to message and enum types. + syntax: string indicating syntax of the file ("proto2" or "proto3") + + Returns: + The added descriptor. + """ + + if package: + desc_name = '.'.join((package, desc_proto.name)) + else: + desc_name = desc_proto.name + + if file_desc is None: + file_name = None + else: + file_name = file_desc.name + + if scope is None: + scope = {} + + nested = [ + self._ConvertMessageDescriptor( + nested, desc_name, file_desc, scope, syntax) + for nested in desc_proto.nested_type] + enums = [ + self._ConvertEnumDescriptor(enum, desc_name, file_desc, None, + scope, False) + for enum in desc_proto.enum_type] + fields = [self._MakeFieldDescriptor(field, desc_name, index, file_desc) + for index, field in enumerate(desc_proto.field)] + extensions = [ + self._MakeFieldDescriptor(extension, desc_name, index, file_desc, + is_extension=True) + for index, extension in enumerate(desc_proto.extension)] + oneofs = [ + # pylint: disable=g-complex-comprehension + descriptor.OneofDescriptor( + desc.name, + '.'.join((desc_name, desc.name)), + index, + None, + [], + _OptionsOrNone(desc), + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + for index, desc in enumerate(desc_proto.oneof_decl) + ] + extension_ranges = [(r.start, r.end) for r in desc_proto.extension_range] + if extension_ranges: + is_extendable = True + else: + is_extendable = False + desc = descriptor.Descriptor( + name=desc_proto.name, + full_name=desc_name, + filename=file_name, + containing_type=None, + fields=fields, + oneofs=oneofs, + nested_types=nested, + enum_types=enums, + extensions=extensions, + options=_OptionsOrNone(desc_proto), + is_extendable=is_extendable, + extension_ranges=extension_ranges, + file=file_desc, + serialized_start=None, + serialized_end=None, + is_map_entry=desc_proto.options.map_entry, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key, + ) + for nested in desc.nested_types: + nested.containing_type = desc + for enum in desc.enum_types: + enum.containing_type = desc + for field_index, field_desc in enumerate(desc_proto.field): + if field_desc.HasField('oneof_index'): + oneof_index = field_desc.oneof_index + oneofs[oneof_index].fields.append(fields[field_index]) + fields[field_index].containing_oneof = oneofs[oneof_index] + + scope[_PrefixWithDot(desc_name)] = desc + self._CheckConflictRegister(desc, desc.full_name, desc.file.name) + self._descriptors[desc_name] = desc + return desc + + def _ConvertEnumDescriptor(self, enum_proto, package=None, file_desc=None, + containing_type=None, scope=None, top_level=False): + """Make a protobuf EnumDescriptor given an EnumDescriptorProto protobuf. + + Args: + enum_proto: The descriptor_pb2.EnumDescriptorProto protobuf message. + package: Optional package name for the new message EnumDescriptor. + file_desc: The file containing the enum descriptor. + containing_type: The type containing this enum. + scope: Scope containing available types. + top_level: If True, the enum is a top level symbol. If False, the enum + is defined inside a message. + + Returns: + The added descriptor + """ + + if package: + enum_name = '.'.join((package, enum_proto.name)) + else: + enum_name = enum_proto.name + + if file_desc is None: + file_name = None + else: + file_name = file_desc.name + + values = [self._MakeEnumValueDescriptor(value, index) + for index, value in enumerate(enum_proto.value)] + desc = descriptor.EnumDescriptor(name=enum_proto.name, + full_name=enum_name, + filename=file_name, + file=file_desc, + values=values, + containing_type=containing_type, + options=_OptionsOrNone(enum_proto), + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + scope['.%s' % enum_name] = desc + self._CheckConflictRegister(desc, desc.full_name, desc.file.name) + self._enum_descriptors[enum_name] = desc + + # Add top level enum values. + if top_level: + for value in values: + full_name = _NormalizeFullyQualifiedName( + '.'.join((package, value.name))) + self._CheckConflictRegister(value, full_name, file_name) + self._top_enum_values[full_name] = value + + return desc + + def _MakeFieldDescriptor(self, field_proto, message_name, index, + file_desc, is_extension=False): + """Creates a field descriptor from a FieldDescriptorProto. + + For message and enum type fields, this method will do a look up + in the pool for the appropriate descriptor for that type. If it + is unavailable, it will fall back to the _source function to + create it. If this type is still unavailable, construction will + fail. + + Args: + field_proto: The proto describing the field. + message_name: The name of the containing message. + index: Index of the field + file_desc: The file containing the field descriptor. + is_extension: Indication that this field is for an extension. + + Returns: + An initialized FieldDescriptor object + """ + + if message_name: + full_name = '.'.join((message_name, field_proto.name)) + else: + full_name = field_proto.name + + if field_proto.json_name: + json_name = field_proto.json_name + else: + json_name = None + + return descriptor.FieldDescriptor( + name=field_proto.name, + full_name=full_name, + index=index, + number=field_proto.number, + type=field_proto.type, + cpp_type=None, + message_type=None, + enum_type=None, + containing_type=None, + label=field_proto.label, + has_default_value=False, + default_value=None, + is_extension=is_extension, + extension_scope=None, + options=_OptionsOrNone(field_proto), + json_name=json_name, + file=file_desc, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + + def _SetAllFieldTypes(self, package, desc_proto, scope): + """Sets all the descriptor's fields's types. + + This method also sets the containing types on any extensions. + + Args: + package: The current package of desc_proto. + desc_proto: The message descriptor to update. + scope: Enclosing scope of available types. + """ + + package = _PrefixWithDot(package) + + main_desc = self._GetTypeFromScope(package, desc_proto.name, scope) + + if package == '.': + nested_package = _PrefixWithDot(desc_proto.name) + else: + nested_package = '.'.join([package, desc_proto.name]) + + for field_proto, field_desc in zip(desc_proto.field, main_desc.fields): + self._SetFieldType(field_proto, field_desc, nested_package, scope) + + for extension_proto, extension_desc in ( + zip(desc_proto.extension, main_desc.extensions)): + extension_desc.containing_type = self._GetTypeFromScope( + nested_package, extension_proto.extendee, scope) + self._SetFieldType(extension_proto, extension_desc, nested_package, scope) + + for nested_type in desc_proto.nested_type: + self._SetAllFieldTypes(nested_package, nested_type, scope) + + def _SetFieldType(self, field_proto, field_desc, package, scope): + """Sets the field's type, cpp_type, message_type and enum_type. + + Args: + field_proto: Data about the field in proto format. + field_desc: The descriptor to modify. + package: The package the field's container is in. + scope: Enclosing scope of available types. + """ + if field_proto.type_name: + desc = self._GetTypeFromScope(package, field_proto.type_name, scope) + else: + desc = None + + if not field_proto.HasField('type'): + if isinstance(desc, descriptor.Descriptor): + field_proto.type = descriptor.FieldDescriptor.TYPE_MESSAGE + else: + field_proto.type = descriptor.FieldDescriptor.TYPE_ENUM + + field_desc.cpp_type = descriptor.FieldDescriptor.ProtoTypeToCppProtoType( + field_proto.type) + + if (field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE + or field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP): + field_desc.message_type = desc + + if field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM: + field_desc.enum_type = desc + + if field_proto.label == descriptor.FieldDescriptor.LABEL_REPEATED: + field_desc.has_default_value = False + field_desc.default_value = [] + elif field_proto.HasField('default_value'): + field_desc.has_default_value = True + if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or + field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT): + field_desc.default_value = float(field_proto.default_value) + elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING: + field_desc.default_value = field_proto.default_value + elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL: + field_desc.default_value = field_proto.default_value.lower() == 'true' + elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM: + field_desc.default_value = field_desc.enum_type.values_by_name[ + field_proto.default_value].number + elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES: + field_desc.default_value = text_encoding.CUnescape( + field_proto.default_value) + elif field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE: + field_desc.default_value = None + else: + # All other types are of the "int" type. + field_desc.default_value = int(field_proto.default_value) + else: + field_desc.has_default_value = False + if (field_proto.type == descriptor.FieldDescriptor.TYPE_DOUBLE or + field_proto.type == descriptor.FieldDescriptor.TYPE_FLOAT): + field_desc.default_value = 0.0 + elif field_proto.type == descriptor.FieldDescriptor.TYPE_STRING: + field_desc.default_value = u'' + elif field_proto.type == descriptor.FieldDescriptor.TYPE_BOOL: + field_desc.default_value = False + elif field_proto.type == descriptor.FieldDescriptor.TYPE_ENUM: + field_desc.default_value = field_desc.enum_type.values[0].number + elif field_proto.type == descriptor.FieldDescriptor.TYPE_BYTES: + field_desc.default_value = b'' + elif field_proto.type == descriptor.FieldDescriptor.TYPE_MESSAGE: + field_desc.default_value = None + elif field_proto.type == descriptor.FieldDescriptor.TYPE_GROUP: + field_desc.default_value = None + else: + # All other types are of the "int" type. + field_desc.default_value = 0 + + field_desc.type = field_proto.type + + def _MakeEnumValueDescriptor(self, value_proto, index): + """Creates a enum value descriptor object from a enum value proto. + + Args: + value_proto: The proto describing the enum value. + index: The index of the enum value. + + Returns: + An initialized EnumValueDescriptor object. + """ + + return descriptor.EnumValueDescriptor( + name=value_proto.name, + index=index, + number=value_proto.number, + options=_OptionsOrNone(value_proto), + type=None, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + + def _MakeServiceDescriptor(self, service_proto, service_index, scope, + package, file_desc): + """Make a protobuf ServiceDescriptor given a ServiceDescriptorProto. + + Args: + service_proto: The descriptor_pb2.ServiceDescriptorProto protobuf message. + service_index: The index of the service in the File. + scope: Dict mapping short and full symbols to message and enum types. + package: Optional package name for the new message EnumDescriptor. + file_desc: The file containing the service descriptor. + + Returns: + The added descriptor. + """ + + if package: + service_name = '.'.join((package, service_proto.name)) + else: + service_name = service_proto.name + + methods = [self._MakeMethodDescriptor(method_proto, service_name, package, + scope, index) + for index, method_proto in enumerate(service_proto.method)] + desc = descriptor.ServiceDescriptor( + name=service_proto.name, + full_name=service_name, + index=service_index, + methods=methods, + options=_OptionsOrNone(service_proto), + file=file_desc, + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + self._CheckConflictRegister(desc, desc.full_name, desc.file.name) + self._service_descriptors[service_name] = desc + return desc + + def _MakeMethodDescriptor(self, method_proto, service_name, package, scope, + index): + """Creates a method descriptor from a MethodDescriptorProto. + + Args: + method_proto: The proto describing the method. + service_name: The name of the containing service. + package: Optional package name to look up for types. + scope: Scope containing available types. + index: Index of the method in the service. + + Returns: + An initialized MethodDescriptor object. + """ + full_name = '.'.join((service_name, method_proto.name)) + input_type = self._GetTypeFromScope( + package, method_proto.input_type, scope) + output_type = self._GetTypeFromScope( + package, method_proto.output_type, scope) + return descriptor.MethodDescriptor( + name=method_proto.name, + full_name=full_name, + index=index, + containing_service=None, + input_type=input_type, + output_type=output_type, + client_streaming=method_proto.client_streaming, + server_streaming=method_proto.server_streaming, + options=_OptionsOrNone(method_proto), + # pylint: disable=protected-access + create_key=descriptor._internal_create_key) + + def _ExtractSymbols(self, descriptors): + """Pulls out all the symbols from descriptor protos. + + Args: + descriptors: The messages to extract descriptors from. + Yields: + A two element tuple of the type name and descriptor object. + """ + + for desc in descriptors: + yield (_PrefixWithDot(desc.full_name), desc) + for symbol in self._ExtractSymbols(desc.nested_types): + yield symbol + for enum in desc.enum_types: + yield (_PrefixWithDot(enum.full_name), enum) + + def _GetDeps(self, dependencies, visited=None): + """Recursively finds dependencies for file protos. + + Args: + dependencies: The names of the files being depended on. + visited: The names of files already found. + + Yields: + Each direct and indirect dependency. + """ + + visited = visited or set() + for dependency in dependencies: + if dependency not in visited: + visited.add(dependency) + dep_desc = self.FindFileByName(dependency) + yield dep_desc + public_files = [d.name for d in dep_desc.public_dependencies] + yield from self._GetDeps(public_files, visited) + + def _GetTypeFromScope(self, package, type_name, scope): + """Finds a given type name in the current scope. + + Args: + package: The package the proto should be located in. + type_name: The name of the type to be found in the scope. + scope: Dict mapping short and full symbols to message and enum types. + + Returns: + The descriptor for the requested type. + """ + if type_name not in scope: + components = _PrefixWithDot(package).split('.') + while components: + possible_match = '.'.join(components + [type_name]) + if possible_match in scope: + type_name = possible_match + break + else: + components.pop(-1) + return scope[type_name] + + +def _PrefixWithDot(name): + return name if name.startswith('.') else '.%s' % name + + +if _USE_C_DESCRIPTORS: + # TODO: This pool could be constructed from Python code, when we + # support a flag like 'use_cpp_generated_pool=True'. + # pylint: disable=protected-access + _DEFAULT = descriptor._message.default_pool +else: + _DEFAULT = DescriptorPool() + + +def Default(): + return _DEFAULT diff --git a/lib/python3.12/site-packages/google/protobuf/duration.py b/lib/python3.12/site-packages/google/protobuf/duration.py new file mode 100644 index 0000000000000000000000000000000000000000..21f17bf71a2f34f412849a24989a6b406df817d0 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/duration.py @@ -0,0 +1,100 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains the Duration helper APIs.""" + +import datetime + +from google.protobuf.duration_pb2 import Duration + + +def from_json_string(value: str) -> Duration: + """Converts a string to Duration. + + Args: + value: A string to be converted. The string must end with 's'. Any + fractional digits (or none) are accepted as long as they fit into + precision. For example: "1s", "1.01s", "1.0000001s", "-3.100s" + + Raises: + ValueError: On parsing problems. + """ + duration = Duration() + duration.FromJsonString(value) + return duration + + +def from_microseconds(micros: float) -> Duration: + """Converts microseconds to Duration.""" + duration = Duration() + duration.FromMicroseconds(micros) + return duration + + +def from_milliseconds(millis: float) -> Duration: + """Converts milliseconds to Duration.""" + duration = Duration() + duration.FromMilliseconds(millis) + return duration + + +def from_nanoseconds(nanos: float) -> Duration: + """Converts nanoseconds to Duration.""" + duration = Duration() + duration.FromNanoseconds(nanos) + return duration + + +def from_seconds(seconds: float) -> Duration: + """Converts seconds to Duration.""" + duration = Duration() + duration.FromSeconds(seconds) + return duration + + +def from_timedelta(td: datetime.timedelta) -> Duration: + """Converts timedelta to Duration.""" + duration = Duration() + duration.FromTimedelta(td) + return duration + + +def to_json_string(duration: Duration) -> str: + """Converts Duration to string format. + + Returns: + A string converted from self. The string format will contains + 3, 6, or 9 fractional digits depending on the precision required to + represent the exact Duration value. For example: "1s", "1.010s", + "1.000000100s", "-3.100s" + """ + return duration.ToJsonString() + + +def to_microseconds(duration: Duration) -> int: + """Converts a Duration to microseconds.""" + return duration.ToMicroseconds() + + +def to_milliseconds(duration: Duration) -> int: + """Converts a Duration to milliseconds.""" + return duration.ToMilliseconds() + + +def to_nanoseconds(duration: Duration) -> int: + """Converts a Duration to nanoseconds.""" + return duration.ToNanoseconds() + + +def to_seconds(duration: Duration) -> int: + """Converts a Duration to seconds.""" + return duration.ToSeconds() + + +def to_timedelta(duration: Duration) -> datetime.timedelta: + """Converts Duration to timedelta.""" + return duration.ToTimedelta() diff --git a/lib/python3.12/site-packages/google/protobuf/duration_pb2.py b/lib/python3.12/site-packages/google/protobuf/duration_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..6d756a425505c2ffb66dcc6a1e755c074aa30bda --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/duration_pb2.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/duration.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/duration.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1egoogle/protobuf/duration.proto\x12\x0fgoogle.protobuf\":\n\x08\x44uration\x12\x18\n\x07seconds\x18\x01 \x01(\x03R\x07seconds\x12\x14\n\x05nanos\x18\x02 \x01(\x05R\x05nanosB\x83\x01\n\x13\x63om.google.protobufB\rDurationProtoP\x01Z1google.golang.org/protobuf/types/known/durationpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.duration_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\rDurationProtoP\001Z1google.golang.org/protobuf/types/known/durationpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_DURATION']._serialized_start=51 + _globals['_DURATION']._serialized_end=109 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/empty_pb2.py b/lib/python3.12/site-packages/google/protobuf/empty_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..109fa29859ac19033e43a28139bc5fccc5dbf76c --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/empty_pb2.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/empty.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/empty.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1bgoogle/protobuf/empty.proto\x12\x0fgoogle.protobuf\"\x07\n\x05\x45mptyB}\n\x13\x63om.google.protobufB\nEmptyProtoP\x01Z.google.golang.org/protobuf/types/known/emptypb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.empty_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\nEmptyProtoP\001Z.google.golang.org/protobuf/types/known/emptypb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_EMPTY']._serialized_start=48 + _globals['_EMPTY']._serialized_end=55 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/field_mask_pb2.py b/lib/python3.12/site-packages/google/protobuf/field_mask_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..1f21c58b4af9bbcc7136b37e60488adb6ec98f39 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/field_mask_pb2.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/field_mask.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/field_mask.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n google/protobuf/field_mask.proto\x12\x0fgoogle.protobuf\"!\n\tFieldMask\x12\x14\n\x05paths\x18\x01 \x03(\tR\x05pathsB\x85\x01\n\x13\x63om.google.protobufB\x0e\x46ieldMaskProtoP\x01Z2google.golang.org/protobuf/types/known/fieldmaskpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.field_mask_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\016FieldMaskProtoP\001Z2google.golang.org/protobuf/types/known/fieldmaskpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_FIELDMASK']._serialized_start=53 + _globals['_FIELDMASK']._serialized_end=86 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__init__.py b/lib/python3.12/site-packages/google/protobuf/internal/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e676e288161e00ae5c266d080ca5018336849a7d --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/__init__.py @@ -0,0 +1,7 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d1461adefea97a3ba8f847ef5f1c38616c173b24 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/api_implementation.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/api_implementation.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9e8002d5b35966c4e800a8fba65615ff62085bff Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/api_implementation.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/builder.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/builder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2ca37148c6facd70108614c6613d647cc2a6afe5 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/builder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/containers.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/containers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8205d74917903c689b8c434cd37d8d9fba938885 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/containers.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/decoder.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/decoder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5aaeeae5b48070ed31616ad2a3375d0eb74abbd8 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/decoder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/encoder.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/encoder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1e52f56e11901b830df1396719b67ae38df40e56 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/encoder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/enum_type_wrapper.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/enum_type_wrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..1019f0b076fd39d049be6b7966a8ba860f9f9eb7 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/enum_type_wrapper.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/extension_dict.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/extension_dict.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dab5bc1135e62df4804420db844bee55875d049f Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/extension_dict.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/field_mask.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/field_mask.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e82d206fbc2e848a9d41eb01d4be8b765e8046a7 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/field_mask.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/message_listener.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/message_listener.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..62324e1459c3d184baf39ceb2d3447850d4069ba Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/message_listener.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/python_edition_defaults.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/python_edition_defaults.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..675a24d62c0a15741e411de24c05ea7788dc6c94 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/python_edition_defaults.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/python_message.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/python_message.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..95909affc52e8e09a8eb2c3d960fa6e6c85e3af4 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/python_message.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/testing_refleaks.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/testing_refleaks.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..50e1a464213ae4b07563c72d0b05378876a45a53 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/testing_refleaks.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/type_checkers.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/type_checkers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e82e436ce6b03472ac1bd7cbc6d6338dbcb5e47f Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/type_checkers.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/well_known_types.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/well_known_types.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d56459aad7cf294d0b8c8d01c342fc2190424f2f Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/well_known_types.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/wire_format.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/wire_format.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d8fc743cb00b573a0295d64c3097ba5118898bde Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/internal/__pycache__/wire_format.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/internal/api_implementation.py b/lib/python3.12/site-packages/google/protobuf/internal/api_implementation.py new file mode 100644 index 0000000000000000000000000000000000000000..f4346bce945fc943d39394012e1817c49a57ed47 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/api_implementation.py @@ -0,0 +1,136 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Determine which implementation of the protobuf API is used in this process. +""" + +import importlib +import os +import sys +import warnings + +_GOOGLE3_PYTHON_UPB_DEFAULT = True + + +def _ApiVersionToImplementationType(api_version): + if api_version == 2: + return 'cpp' + if api_version == 1: + raise ValueError('api_version=1 is no longer supported.') + if api_version == 0: + return 'python' + return None + + +_implementation_type = None +try: + # pylint: disable=g-import-not-at-top + from google.protobuf.internal import _api_implementation + # The compile-time constants in the _api_implementation module can be used to + # switch to a certain implementation of the Python API at build time. + _implementation_type = _ApiVersionToImplementationType( + _api_implementation.api_version) +except ImportError: + pass # Unspecified by compiler flags. + + +def _CanImport(mod_name): + try: + mod = importlib.import_module(mod_name) + # Work around a known issue in the classic bootstrap .par import hook. + if not mod: + raise ImportError(mod_name + ' import succeeded but was None') + return True + except ImportError: + return False + + +if _implementation_type is None: + if _CanImport('google._upb._message'): + _implementation_type = 'upb' + elif _CanImport('google.protobuf.pyext._message'): + _implementation_type = 'cpp' + else: + _implementation_type = 'python' + + +# This environment variable can be used to switch to a certain implementation +# of the Python API, overriding the compile-time constants in the +# _api_implementation module. Right now only 'python', 'cpp' and 'upb' are +# valid values. Any other value will raise error. +_implementation_type = os.getenv('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION', + _implementation_type) + +if _implementation_type not in ('python', 'cpp', 'upb'): + raise ValueError('PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION {0} is not ' + 'supported. Please set to \'python\', \'cpp\' or ' + '\'upb\'.'.format(_implementation_type)) + +if 'PyPy' in sys.version and _implementation_type == 'cpp': + warnings.warn('PyPy does not work yet with cpp protocol buffers. ' + 'Falling back to the python implementation.') + _implementation_type = 'python' + +_c_module = None + +if _implementation_type == 'cpp': + try: + # pylint: disable=g-import-not-at-top + from google.protobuf.pyext import _message + sys.modules['google3.net.proto2.python.internal.cpp._message'] = _message + _c_module = _message + del _message + except ImportError: + # TODO: fail back to python + warnings.warn( + 'Selected implementation cpp is not available.') + pass + +if _implementation_type == 'upb': + try: + # pylint: disable=g-import-not-at-top + from google._upb import _message + _c_module = _message + del _message + except ImportError: + warnings.warn('Selected implementation upb is not available. ' + 'Falling back to the python implementation.') + _implementation_type = 'python' + pass + +# Detect if serialization should be deterministic by default +try: + # The presence of this module in a build allows the proto implementation to + # be upgraded merely via build deps. + # + # NOTE: Merely importing this automatically enables deterministic proto + # serialization for C++ code, but we still need to export it as a boolean so + # that we can do the same for `_implementation_type == 'python'`. + # + # NOTE2: It is possible for C++ code to enable deterministic serialization by + # default _without_ affecting Python code, if the C++ implementation is not in + # use by this module. That is intended behavior, so we don't actually expose + # this boolean outside of this module. + # + # pylint: disable=g-import-not-at-top,unused-import + from google.protobuf import enable_deterministic_proto_serialization + _python_deterministic_proto_serialization = True +except ImportError: + _python_deterministic_proto_serialization = False + + +# Usage of this function is discouraged. Clients shouldn't care which +# implementation of the API is in use. Note that there is no guarantee +# that differences between APIs will be maintained. +# Please don't use this function if possible. +def Type(): + return _implementation_type + + +# For internal use only +def IsPythonDefaultSerializationDeterministic(): + return _python_deterministic_proto_serialization diff --git a/lib/python3.12/site-packages/google/protobuf/internal/builder.py b/lib/python3.12/site-packages/google/protobuf/internal/builder.py new file mode 100644 index 0000000000000000000000000000000000000000..9859e490e6415c3d6222f131631062156d2a4890 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/builder.py @@ -0,0 +1,118 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Builds descriptors, message classes and services for generated _pb2.py. + +This file is only called in python generated _pb2.py files. It builds +descriptors, message classes and services that users can directly use +in generated code. +""" + +__author__ = 'jieluo@google.com (Jie Luo)' + +from google.protobuf.internal import enum_type_wrapper +from google.protobuf.internal import python_message +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database + +_sym_db = _symbol_database.Default() + + +def BuildMessageAndEnumDescriptors(file_des, module): + """Builds message and enum descriptors. + + Args: + file_des: FileDescriptor of the .proto file + module: Generated _pb2 module + """ + + def BuildNestedDescriptors(msg_des, prefix): + for (name, nested_msg) in msg_des.nested_types_by_name.items(): + module_name = prefix + name.upper() + module[module_name] = nested_msg + BuildNestedDescriptors(nested_msg, module_name + '_') + for enum_des in msg_des.enum_types: + module[prefix + enum_des.name.upper()] = enum_des + + for (name, msg_des) in file_des.message_types_by_name.items(): + module_name = '_' + name.upper() + module[module_name] = msg_des + BuildNestedDescriptors(msg_des, module_name + '_') + + +def BuildTopDescriptorsAndMessages(file_des, module_name, module): + """Builds top level descriptors and message classes. + + Args: + file_des: FileDescriptor of the .proto file + module_name: str, the name of generated _pb2 module + module: Generated _pb2 module + """ + + def BuildMessage(msg_des, prefix): + create_dict = {} + for (name, nested_msg) in msg_des.nested_types_by_name.items(): + create_dict[name] = BuildMessage(nested_msg, prefix + msg_des.name + '.') + create_dict['DESCRIPTOR'] = msg_des + create_dict['__module__'] = module_name + create_dict['__qualname__'] = prefix + msg_des.name + message_class = _reflection.GeneratedProtocolMessageType( + msg_des.name, (_message.Message,), create_dict) + _sym_db.RegisterMessage(message_class) + return message_class + + # top level enums + for (name, enum_des) in file_des.enum_types_by_name.items(): + module['_' + name.upper()] = enum_des + module[name] = enum_type_wrapper.EnumTypeWrapper(enum_des) + for enum_value in enum_des.values: + module[enum_value.name] = enum_value.number + + # top level extensions + for (name, extension_des) in file_des.extensions_by_name.items(): + module[name.upper() + '_FIELD_NUMBER'] = extension_des.number + module[name] = extension_des + + # services + for (name, service) in file_des.services_by_name.items(): + module['_' + name.upper()] = service + + # Build messages. + for (name, msg_des) in file_des.message_types_by_name.items(): + module[name] = BuildMessage(msg_des, '') + + +def AddHelpersToExtensions(file_des): + """no-op to keep old generated code work with new runtime. + + Args: + file_des: FileDescriptor of the .proto file + """ + # TODO: Remove this on-op + return + + +def BuildServices(file_des, module_name, module): + """Builds services classes and services stub class. + + Args: + file_des: FileDescriptor of the .proto file + module_name: str, the name of generated _pb2 module + module: Generated _pb2 module + """ + # pylint: disable=g-import-not-at-top + from google.protobuf import service_reflection + # pylint: enable=g-import-not-at-top + for (name, service) in file_des.services_by_name.items(): + module[name] = service_reflection.GeneratedServiceType( + name, (), + dict(DESCRIPTOR=service, __module__=module_name)) + stub_name = name + '_Stub' + module[stub_name] = service_reflection.GeneratedServiceStubType( + stub_name, (module[name],), + dict(DESCRIPTOR=service, __module__=module_name)) diff --git a/lib/python3.12/site-packages/google/protobuf/internal/containers.py b/lib/python3.12/site-packages/google/protobuf/internal/containers.py new file mode 100644 index 0000000000000000000000000000000000000000..7298bc5c7a33dad27b1e933fc04c1435ae4b6986 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/containers.py @@ -0,0 +1,690 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains container classes to represent different protocol buffer types. + +This file defines container classes which represent categories of protocol +buffer field types which need extra maintenance. Currently these categories +are: + +- Repeated scalar fields - These are all repeated fields which aren't + composite (e.g. they are of simple types like int32, string, etc). +- Repeated composite fields - Repeated fields which are composite. This + includes groups and nested messages. +""" + +import collections.abc +import copy +import pickle +from typing import ( + Any, + Iterable, + Iterator, + List, + MutableMapping, + MutableSequence, + NoReturn, + Optional, + Sequence, + TypeVar, + Union, + overload, +) + + +_T = TypeVar('_T') +_K = TypeVar('_K') +_V = TypeVar('_V') + + +class BaseContainer(Sequence[_T]): + """Base container class.""" + + # Minimizes memory usage and disallows assignment to other attributes. + __slots__ = ['_message_listener', '_values'] + + def __init__(self, message_listener: Any) -> None: + """ + Args: + message_listener: A MessageListener implementation. + The RepeatedScalarFieldContainer will call this object's + Modified() method when it is modified. + """ + self._message_listener = message_listener + self._values = [] + + @overload + def __getitem__(self, key: int) -> _T: + ... + + @overload + def __getitem__(self, key: slice) -> List[_T]: + ... + + def __getitem__(self, key): + """Retrieves item by the specified key.""" + return self._values[key] + + def __len__(self) -> int: + """Returns the number of elements in the container.""" + return len(self._values) + + def __ne__(self, other: Any) -> bool: + """Checks if another instance isn't equal to this one.""" + # The concrete classes should define __eq__. + return not self == other + + __hash__ = None + + def __repr__(self) -> str: + return repr(self._values) + + def sort(self, *args, **kwargs) -> None: + # Continue to support the old sort_function keyword argument. + # This is expected to be a rare occurrence, so use LBYL to avoid + # the overhead of actually catching KeyError. + if 'sort_function' in kwargs: + kwargs['cmp'] = kwargs.pop('sort_function') + self._values.sort(*args, **kwargs) + + def reverse(self) -> None: + self._values.reverse() + + +# TODO: Remove this. BaseContainer does *not* conform to +# MutableSequence, only its subclasses do. +collections.abc.MutableSequence.register(BaseContainer) + + +class RepeatedScalarFieldContainer(BaseContainer[_T], MutableSequence[_T]): + """Simple, type-checked, list-like container for holding repeated scalars.""" + + # Disallows assignment to other attributes. + __slots__ = ['_type_checker'] + + def __init__( + self, + message_listener: Any, + type_checker: Any, + ) -> None: + """Args: + + message_listener: A MessageListener implementation. The + RepeatedScalarFieldContainer will call this object's Modified() method + when it is modified. + type_checker: A type_checkers.ValueChecker instance to run on elements + inserted into this container. + """ + super().__init__(message_listener) + self._type_checker = type_checker + + def append(self, value: _T) -> None: + """Appends an item to the list. Similar to list.append().""" + self._values.append(self._type_checker.CheckValue(value)) + if not self._message_listener.dirty: + self._message_listener.Modified() + + def insert(self, key: int, value: _T) -> None: + """Inserts the item at the specified position. Similar to list.insert().""" + self._values.insert(key, self._type_checker.CheckValue(value)) + if not self._message_listener.dirty: + self._message_listener.Modified() + + def extend(self, elem_seq: Iterable[_T]) -> None: + """Extends by appending the given iterable. Similar to list.extend().""" + elem_seq_iter = iter(elem_seq) + new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter] + if new_values: + self._values.extend(new_values) + self._message_listener.Modified() + + def MergeFrom( + self, + other: Union['RepeatedScalarFieldContainer[_T]', Iterable[_T]], + ) -> None: + """Appends the contents of another repeated field of the same type to this + one. We do not check the types of the individual fields. + """ + self._values.extend(other) + self._message_listener.Modified() + + def remove(self, elem: _T): + """Removes an item from the list. Similar to list.remove().""" + self._values.remove(elem) + self._message_listener.Modified() + + def pop(self, key: Optional[int] = -1) -> _T: + """Removes and returns an item at a given index. Similar to list.pop().""" + value = self._values[key] + self.__delitem__(key) + return value + + @overload + def __setitem__(self, key: int, value: _T) -> None: + ... + + @overload + def __setitem__(self, key: slice, value: Iterable[_T]) -> None: + ... + + def __setitem__(self, key, value) -> None: + """Sets the item on the specified position.""" + if isinstance(key, slice): + if key.step is not None: + raise ValueError('Extended slices not supported') + self._values[key] = map(self._type_checker.CheckValue, value) + self._message_listener.Modified() + else: + self._values[key] = self._type_checker.CheckValue(value) + self._message_listener.Modified() + + def __delitem__(self, key: Union[int, slice]) -> None: + """Deletes the item at the specified position.""" + del self._values[key] + self._message_listener.Modified() + + def __eq__(self, other: Any) -> bool: + """Compares the current instance with another one.""" + if self is other: + return True + # Special case for the same type which should be common and fast. + if isinstance(other, self.__class__): + return other._values == self._values + # We are presumably comparing against some other sequence type. + return other == self._values + + def __deepcopy__( + self, + unused_memo: Any = None, + ) -> 'RepeatedScalarFieldContainer[_T]': + clone = RepeatedScalarFieldContainer( + copy.deepcopy(self._message_listener), self._type_checker) + clone.MergeFrom(self) + return clone + + def __reduce__(self, **kwargs) -> NoReturn: + raise pickle.PickleError( + "Can't pickle repeated scalar fields, convert to list first") + + +# TODO: Constrain T to be a subtype of Message. +class RepeatedCompositeFieldContainer(BaseContainer[_T], MutableSequence[_T]): + """Simple, list-like container for holding repeated composite fields.""" + + # Disallows assignment to other attributes. + __slots__ = ['_message_descriptor'] + + def __init__(self, message_listener: Any, message_descriptor: Any) -> None: + """ + Note that we pass in a descriptor instead of the generated directly, + since at the time we construct a _RepeatedCompositeFieldContainer we + haven't yet necessarily initialized the type that will be contained in the + container. + + Args: + message_listener: A MessageListener implementation. + The RepeatedCompositeFieldContainer will call this object's + Modified() method when it is modified. + message_descriptor: A Descriptor instance describing the protocol type + that should be present in this container. We'll use the + _concrete_class field of this descriptor when the client calls add(). + """ + super().__init__(message_listener) + self._message_descriptor = message_descriptor + + def add(self, **kwargs: Any) -> _T: + """Adds a new element at the end of the list and returns it. Keyword + arguments may be used to initialize the element. + """ + new_element = self._message_descriptor._concrete_class(**kwargs) + new_element._SetListener(self._message_listener) + self._values.append(new_element) + if not self._message_listener.dirty: + self._message_listener.Modified() + return new_element + + def append(self, value: _T) -> None: + """Appends one element by copying the message.""" + new_element = self._message_descriptor._concrete_class() + new_element._SetListener(self._message_listener) + new_element.CopyFrom(value) + self._values.append(new_element) + if not self._message_listener.dirty: + self._message_listener.Modified() + + def insert(self, key: int, value: _T) -> None: + """Inserts the item at the specified position by copying.""" + new_element = self._message_descriptor._concrete_class() + new_element._SetListener(self._message_listener) + new_element.CopyFrom(value) + self._values.insert(key, new_element) + if not self._message_listener.dirty: + self._message_listener.Modified() + + def extend(self, elem_seq: Iterable[_T]) -> None: + """Extends by appending the given sequence of elements of the same type + + as this one, copying each individual message. + """ + message_class = self._message_descriptor._concrete_class + listener = self._message_listener + values = self._values + for message in elem_seq: + new_element = message_class() + new_element._SetListener(listener) + new_element.MergeFrom(message) + values.append(new_element) + listener.Modified() + + def MergeFrom( + self, + other: Union['RepeatedCompositeFieldContainer[_T]', Iterable[_T]], + ) -> None: + """Appends the contents of another repeated field of the same type to this + one, copying each individual message. + """ + self.extend(other) + + def remove(self, elem: _T) -> None: + """Removes an item from the list. Similar to list.remove().""" + self._values.remove(elem) + self._message_listener.Modified() + + def pop(self, key: Optional[int] = -1) -> _T: + """Removes and returns an item at a given index. Similar to list.pop().""" + value = self._values[key] + self.__delitem__(key) + return value + + @overload + def __setitem__(self, key: int, value: _T) -> None: + ... + + @overload + def __setitem__(self, key: slice, value: Iterable[_T]) -> None: + ... + + def __setitem__(self, key, value): + # This method is implemented to make RepeatedCompositeFieldContainer + # structurally compatible with typing.MutableSequence. It is + # otherwise unsupported and will always raise an error. + raise TypeError( + f'{self.__class__.__name__} object does not support item assignment') + + def __delitem__(self, key: Union[int, slice]) -> None: + """Deletes the item at the specified position.""" + del self._values[key] + self._message_listener.Modified() + + def __eq__(self, other: Any) -> bool: + """Compares the current instance with another one.""" + if self is other: + return True + if not isinstance(other, self.__class__): + raise TypeError('Can only compare repeated composite fields against ' + 'other repeated composite fields.') + return self._values == other._values + + +class ScalarMap(MutableMapping[_K, _V]): + """Simple, type-checked, dict-like container for holding repeated scalars.""" + + # Disallows assignment to other attributes. + __slots__ = ['_key_checker', '_value_checker', '_values', '_message_listener', + '_entry_descriptor'] + + def __init__( + self, + message_listener: Any, + key_checker: Any, + value_checker: Any, + entry_descriptor: Any, + ) -> None: + """ + Args: + message_listener: A MessageListener implementation. + The ScalarMap will call this object's Modified() method when it + is modified. + key_checker: A type_checkers.ValueChecker instance to run on keys + inserted into this container. + value_checker: A type_checkers.ValueChecker instance to run on values + inserted into this container. + entry_descriptor: The MessageDescriptor of a map entry: key and value. + """ + self._message_listener = message_listener + self._key_checker = key_checker + self._value_checker = value_checker + self._entry_descriptor = entry_descriptor + self._values = {} + + def __getitem__(self, key: _K) -> _V: + try: + return self._values[key] + except KeyError: + key = self._key_checker.CheckValue(key) + val = self._value_checker.DefaultValue() + self._values[key] = val + return val + + def __contains__(self, item: _K) -> bool: + # We check the key's type to match the strong-typing flavor of the API. + # Also this makes it easier to match the behavior of the C++ implementation. + self._key_checker.CheckValue(item) + return item in self._values + + @overload + def get(self, key: _K) -> Optional[_V]: + ... + + @overload + def get(self, key: _K, default: _T) -> Union[_V, _T]: + ... + + # We need to override this explicitly, because our defaultdict-like behavior + # will make the default implementation (from our base class) always insert + # the key. + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + + def __setitem__(self, key: _K, value: _V) -> _T: + checked_key = self._key_checker.CheckValue(key) + checked_value = self._value_checker.CheckValue(value) + self._values[checked_key] = checked_value + self._message_listener.Modified() + + def __delitem__(self, key: _K) -> None: + del self._values[key] + self._message_listener.Modified() + + def __len__(self) -> int: + return len(self._values) + + def __iter__(self) -> Iterator[_K]: + return iter(self._values) + + def __repr__(self) -> str: + return repr(self._values) + + def setdefault(self, key: _K, value: Optional[_V] = None) -> _V: + if value == None: + raise ValueError('The value for scalar map setdefault must be set.') + if key not in self._values: + self.__setitem__(key, value) + return self[key] + + def MergeFrom(self, other: 'ScalarMap[_K, _V]') -> None: + self._values.update(other._values) + self._message_listener.Modified() + + def InvalidateIterators(self) -> None: + # It appears that the only way to reliably invalidate iterators to + # self._values is to ensure that its size changes. + original = self._values + self._values = original.copy() + original[None] = None + + # This is defined in the abstract base, but we can do it much more cheaply. + def clear(self) -> None: + self._values.clear() + self._message_listener.Modified() + + def GetEntryClass(self) -> Any: + return self._entry_descriptor._concrete_class + + +class MessageMap(MutableMapping[_K, _V]): + """Simple, type-checked, dict-like container for with submessage values.""" + + # Disallows assignment to other attributes. + __slots__ = ['_key_checker', '_values', '_message_listener', + '_message_descriptor', '_entry_descriptor'] + + def __init__( + self, + message_listener: Any, + message_descriptor: Any, + key_checker: Any, + entry_descriptor: Any, + ) -> None: + """ + Args: + message_listener: A MessageListener implementation. + The ScalarMap will call this object's Modified() method when it + is modified. + key_checker: A type_checkers.ValueChecker instance to run on keys + inserted into this container. + value_checker: A type_checkers.ValueChecker instance to run on values + inserted into this container. + entry_descriptor: The MessageDescriptor of a map entry: key and value. + """ + self._message_listener = message_listener + self._message_descriptor = message_descriptor + self._key_checker = key_checker + self._entry_descriptor = entry_descriptor + self._values = {} + + def __getitem__(self, key: _K) -> _V: + key = self._key_checker.CheckValue(key) + try: + return self._values[key] + except KeyError: + new_element = self._message_descriptor._concrete_class() + new_element._SetListener(self._message_listener) + self._values[key] = new_element + self._message_listener.Modified() + return new_element + + def get_or_create(self, key: _K) -> _V: + """get_or_create() is an alias for getitem (ie. map[key]). + + Args: + key: The key to get or create in the map. + + This is useful in cases where you want to be explicit that the call is + mutating the map. This can avoid lint errors for statements like this + that otherwise would appear to be pointless statements: + + msg.my_map[key] + """ + return self[key] + + @overload + def get(self, key: _K) -> Optional[_V]: + ... + + @overload + def get(self, key: _K, default: _T) -> Union[_V, _T]: + ... + + # We need to override this explicitly, because our defaultdict-like behavior + # will make the default implementation (from our base class) always insert + # the key. + def get(self, key, default=None): + if key in self: + return self[key] + else: + return default + + def __contains__(self, item: _K) -> bool: + item = self._key_checker.CheckValue(item) + return item in self._values + + def __setitem__(self, key: _K, value: _V) -> NoReturn: + raise ValueError('May not set values directly, call my_map[key].foo = 5') + + def __delitem__(self, key: _K) -> None: + key = self._key_checker.CheckValue(key) + del self._values[key] + self._message_listener.Modified() + + def __len__(self) -> int: + return len(self._values) + + def __iter__(self) -> Iterator[_K]: + return iter(self._values) + + def __repr__(self) -> str: + return repr(self._values) + + def setdefault(self, key: _K, value: Optional[_V] = None) -> _V: + raise NotImplementedError( + 'Set message map value directly is not supported, call' + ' my_map[key].foo = 5' + ) + + def MergeFrom(self, other: 'MessageMap[_K, _V]') -> None: + # pylint: disable=protected-access + for key in other._values: + # According to documentation: "When parsing from the wire or when merging, + # if there are duplicate map keys the last key seen is used". + if key in self: + del self[key] + self[key].CopyFrom(other[key]) + # self._message_listener.Modified() not required here, because + # mutations to submessages already propagate. + + def InvalidateIterators(self) -> None: + # It appears that the only way to reliably invalidate iterators to + # self._values is to ensure that its size changes. + original = self._values + self._values = original.copy() + original[None] = None + + # This is defined in the abstract base, but we can do it much more cheaply. + def clear(self) -> None: + self._values.clear() + self._message_listener.Modified() + + def GetEntryClass(self) -> Any: + return self._entry_descriptor._concrete_class + + +class _UnknownField: + """A parsed unknown field.""" + + # Disallows assignment to other attributes. + __slots__ = ['_field_number', '_wire_type', '_data'] + + def __init__(self, field_number, wire_type, data): + self._field_number = field_number + self._wire_type = wire_type + self._data = data + return + + def __lt__(self, other): + # pylint: disable=protected-access + return self._field_number < other._field_number + + def __eq__(self, other): + if self is other: + return True + # pylint: disable=protected-access + return (self._field_number == other._field_number and + self._wire_type == other._wire_type and + self._data == other._data) + + +class UnknownFieldRef: # pylint: disable=missing-class-docstring + + def __init__(self, parent, index): + self._parent = parent + self._index = index + + def _check_valid(self): + if not self._parent: + raise ValueError('UnknownField does not exist. ' + 'The parent message might be cleared.') + if self._index >= len(self._parent): + raise ValueError('UnknownField does not exist. ' + 'The parent message might be cleared.') + + @property + def field_number(self): + self._check_valid() + # pylint: disable=protected-access + return self._parent._internal_get(self._index)._field_number + + @property + def wire_type(self): + self._check_valid() + # pylint: disable=protected-access + return self._parent._internal_get(self._index)._wire_type + + @property + def data(self): + self._check_valid() + # pylint: disable=protected-access + return self._parent._internal_get(self._index)._data + + +class UnknownFieldSet: + """UnknownField container""" + + # Disallows assignment to other attributes. + __slots__ = ['_values'] + + def __init__(self): + self._values = [] + + def __getitem__(self, index): + if self._values is None: + raise ValueError('UnknownFields does not exist. ' + 'The parent message might be cleared.') + size = len(self._values) + if index < 0: + index += size + if index < 0 or index >= size: + raise IndexError('index %d out of range'.index) + + return UnknownFieldRef(self, index) + + def _internal_get(self, index): + return self._values[index] + + def __len__(self): + if self._values is None: + raise ValueError('UnknownFields does not exist. ' + 'The parent message might be cleared.') + return len(self._values) + + def _add(self, field_number, wire_type, data): + unknown_field = _UnknownField(field_number, wire_type, data) + self._values.append(unknown_field) + return unknown_field + + def __iter__(self): + for i in range(len(self)): + yield UnknownFieldRef(self, i) + + def _extend(self, other): + if other is None: + return + # pylint: disable=protected-access + self._values.extend(other._values) + + def __eq__(self, other): + if self is other: + return True + # Sort unknown fields because their order shouldn't + # affect equality test. + values = list(self._values) + if other is None: + return not values + values.sort() + # pylint: disable=protected-access + other_values = sorted(other._values) + return values == other_values + + def _clear(self): + for value in self._values: + # pylint: disable=protected-access + if isinstance(value._data, UnknownFieldSet): + value._data._clear() # pylint: disable=protected-access + self._values = None diff --git a/lib/python3.12/site-packages/google/protobuf/internal/decoder.py b/lib/python3.12/site-packages/google/protobuf/internal/decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..dbf848bf31083d0c4dfbc32fed58e701049f6ede --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/decoder.py @@ -0,0 +1,1066 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Code for decoding protocol buffer primitives. + +This code is very similar to encoder.py -- read the docs for that module first. + +A "decoder" is a function with the signature: + Decode(buffer, pos, end, message, field_dict) +The arguments are: + buffer: The string containing the encoded message. + pos: The current position in the string. + end: The position in the string where the current message ends. May be + less than len(buffer) if we're reading a sub-message. + message: The message object into which we're parsing. + field_dict: message._fields (avoids a hashtable lookup). +The decoder reads the field and stores it into field_dict, returning the new +buffer position. A decoder for a repeated field may proactively decode all of +the elements of that field, if they appear consecutively. + +Note that decoders may throw any of the following: + IndexError: Indicates a truncated message. + struct.error: Unpacking of a fixed-width field failed. + message.DecodeError: Other errors. + +Decoders are expected to raise an exception if they are called with pos > end. +This allows callers to be lax about bounds checking: it's fineto read past +"end" as long as you are sure that someone else will notice and throw an +exception later on. + +Something up the call stack is expected to catch IndexError and struct.error +and convert them to message.DecodeError. + +Decoders are constructed using decoder constructors with the signature: + MakeDecoder(field_number, is_repeated, is_packed, key, new_default) +The arguments are: + field_number: The field number of the field we want to decode. + is_repeated: Is the field a repeated field? (bool) + is_packed: Is the field a packed field? (bool) + key: The key to use when looking up the field within field_dict. + (This is actually the FieldDescriptor but nothing in this + file should depend on that.) + new_default: A function which takes a message object as a parameter and + returns a new instance of the default value for this field. + (This is called for repeated fields and sub-messages, when an + instance does not already exist.) + +As with encoders, we define a decoder constructor for every type of field. +Then, for every field of every message class we construct an actual decoder. +That decoder goes into a dict indexed by tag, so when we decode a message +we repeatedly read a tag, look up the corresponding decoder, and invoke it. +""" + +__author__ = 'kenton@google.com (Kenton Varda)' + +import math +import numbers +import struct + +from google.protobuf import message +from google.protobuf.internal import containers +from google.protobuf.internal import encoder +from google.protobuf.internal import wire_format + + +# This is not for optimization, but rather to avoid conflicts with local +# variables named "message". +_DecodeError = message.DecodeError + + +def IsDefaultScalarValue(value): + """Returns whether or not a scalar value is the default value of its type. + + Specifically, this should be used to determine presence of implicit-presence + fields, where we disallow custom defaults. + + Args: + value: A scalar value to check. + + Returns: + True if the value is equivalent to a default value, False otherwise. + """ + if isinstance(value, numbers.Number) and math.copysign(1.0, value) < 0: + # Special case for negative zero, where "truthiness" fails to give the right + # answer. + return False + + # Normally, we can just use Python's boolean conversion. + return not value + + +def _VarintDecoder(mask, result_type): + """Return an encoder for a basic varint value (does not include tag). + + Decoded values will be bitwise-anded with the given mask before being + returned, e.g. to limit them to 32 bits. The returned decoder does not + take the usual "end" parameter -- the caller is expected to do bounds checking + after the fact (often the caller can defer such checking until later). The + decoder returns a (value, new_pos) pair. + """ + + def DecodeVarint(buffer, pos: int=None): + result = 0 + shift = 0 + while 1: + if pos is None: + # Read from BytesIO + try: + b = buffer.read(1)[0] + except IndexError as e: + if shift == 0: + # End of BytesIO. + return None + else: + raise ValueError('Fail to read varint %s' % str(e)) + else: + b = buffer[pos] + pos += 1 + result |= ((b & 0x7f) << shift) + if not (b & 0x80): + result &= mask + result = result_type(result) + return result if pos is None else (result, pos) + shift += 7 + if shift >= 64: + raise _DecodeError('Too many bytes when decoding varint.') + + return DecodeVarint + + +def _SignedVarintDecoder(bits, result_type): + """Like _VarintDecoder() but decodes signed values.""" + + signbit = 1 << (bits - 1) + mask = (1 << bits) - 1 + + def DecodeVarint(buffer, pos): + result = 0 + shift = 0 + while 1: + b = buffer[pos] + result |= ((b & 0x7f) << shift) + pos += 1 + if not (b & 0x80): + result &= mask + result = (result ^ signbit) - signbit + result = result_type(result) + return (result, pos) + shift += 7 + if shift >= 64: + raise _DecodeError('Too many bytes when decoding varint.') + return DecodeVarint + +# All 32-bit and 64-bit values are represented as int. +_DecodeVarint = _VarintDecoder((1 << 64) - 1, int) +_DecodeSignedVarint = _SignedVarintDecoder(64, int) + +# Use these versions for values which must be limited to 32 bits. +_DecodeVarint32 = _VarintDecoder((1 << 32) - 1, int) +_DecodeSignedVarint32 = _SignedVarintDecoder(32, int) + + +def ReadTag(buffer, pos): + """Read a tag from the memoryview, and return a (tag_bytes, new_pos) tuple. + + We return the raw bytes of the tag rather than decoding them. The raw + bytes can then be used to look up the proper decoder. This effectively allows + us to trade some work that would be done in pure-python (decoding a varint) + for work that is done in C (searching for a byte string in a hash table). + In a low-level language it would be much cheaper to decode the varint and + use that, but not in Python. + + Args: + buffer: memoryview object of the encoded bytes + pos: int of the current position to start from + + Returns: + Tuple[bytes, int] of the tag data and new position. + """ + start = pos + while buffer[pos] & 0x80: + pos += 1 + pos += 1 + + tag_bytes = buffer[start:pos].tobytes() + return tag_bytes, pos + + +def DecodeTag(tag_bytes): + """Decode a tag from the bytes. + + Args: + tag_bytes: the bytes of the tag + + Returns: + Tuple[int, int] of the tag field number and wire type. + """ + (tag, _) = _DecodeVarint(tag_bytes, 0) + return wire_format.UnpackTag(tag) + + +# -------------------------------------------------------------------- + + +def _SimpleDecoder(wire_type, decode_value): + """Return a constructor for a decoder for fields of a particular type. + + Args: + wire_type: The field's wire type. + decode_value: A function which decodes an individual value, e.g. + _DecodeVarint() + """ + + def SpecificDecoder(field_number, is_repeated, is_packed, key, new_default, + clear_if_default=False): + if is_packed: + local_DecodeVarint = _DecodeVarint + def DecodePackedField( + buffer, pos, end, message, field_dict, current_depth=0 + ): + del current_depth # unused + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + (endpoint, pos) = local_DecodeVarint(buffer, pos) + endpoint += pos + if endpoint > end: + raise _DecodeError('Truncated message.') + while pos < endpoint: + (element, pos) = decode_value(buffer, pos) + value.append(element) + if pos > endpoint: + del value[-1] # Discard corrupt value. + raise _DecodeError('Packed element was truncated.') + return pos + + return DecodePackedField + elif is_repeated: + tag_bytes = encoder.TagBytes(field_number, wire_type) + tag_len = len(tag_bytes) + def DecodeRepeatedField( + buffer, pos, end, message, field_dict, current_depth=0 + ): + del current_depth # unused + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + while 1: + (element, new_pos) = decode_value(buffer, pos) + value.append(element) + # Predict that the next tag is another copy of the same repeated + # field. + pos = new_pos + tag_len + if buffer[new_pos:pos] != tag_bytes or new_pos >= end: + # Prediction failed. Return. + if new_pos > end: + raise _DecodeError('Truncated message.') + return new_pos + + return DecodeRepeatedField + else: + + def DecodeField(buffer, pos, end, message, field_dict, current_depth=0): + del current_depth # unused + (new_value, pos) = decode_value(buffer, pos) + if pos > end: + raise _DecodeError('Truncated message.') + if clear_if_default and IsDefaultScalarValue(new_value): + field_dict.pop(key, None) + else: + field_dict[key] = new_value + return pos + + return DecodeField + + return SpecificDecoder + + +def _ModifiedDecoder(wire_type, decode_value, modify_value): + """Like SimpleDecoder but additionally invokes modify_value on every value + before storing it. Usually modify_value is ZigZagDecode. + """ + + # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but + # not enough to make a significant difference. + + def InnerDecode(buffer, pos): + (result, new_pos) = decode_value(buffer, pos) + return (modify_value(result), new_pos) + return _SimpleDecoder(wire_type, InnerDecode) + + +def _StructPackDecoder(wire_type, format): + """Return a constructor for a decoder for a fixed-width field. + + Args: + wire_type: The field's wire type. + format: The format string to pass to struct.unpack(). + """ + + value_size = struct.calcsize(format) + local_unpack = struct.unpack + + # Reusing _SimpleDecoder is slightly slower than copying a bunch of code, but + # not enough to make a significant difference. + + # Note that we expect someone up-stack to catch struct.error and convert + # it to _DecodeError -- this way we don't have to set up exception- + # handling blocks every time we parse one value. + + def InnerDecode(buffer, pos): + new_pos = pos + value_size + result = local_unpack(format, buffer[pos:new_pos])[0] + return (result, new_pos) + return _SimpleDecoder(wire_type, InnerDecode) + + +def _FloatDecoder(): + """Returns a decoder for a float field. + + This code works around a bug in struct.unpack for non-finite 32-bit + floating-point values. + """ + + local_unpack = struct.unpack + + def InnerDecode(buffer, pos): + """Decode serialized float to a float and new position. + + Args: + buffer: memoryview of the serialized bytes + pos: int, position in the memory view to start at. + + Returns: + Tuple[float, int] of the deserialized float value and new position + in the serialized data. + """ + # We expect a 32-bit value in little-endian byte order. Bit 1 is the sign + # bit, bits 2-9 represent the exponent, and bits 10-32 are the significand. + new_pos = pos + 4 + float_bytes = buffer[pos:new_pos].tobytes() + + # If this value has all its exponent bits set, then it's non-finite. + # In Python 2.4, struct.unpack will convert it to a finite 64-bit value. + # To avoid that, we parse it specially. + if (float_bytes[3:4] in b'\x7F\xFF' and float_bytes[2:3] >= b'\x80'): + # If at least one significand bit is set... + if float_bytes[0:3] != b'\x00\x00\x80': + return (math.nan, new_pos) + # If sign bit is set... + if float_bytes[3:4] == b'\xFF': + return (-math.inf, new_pos) + return (math.inf, new_pos) + + # Note that we expect someone up-stack to catch struct.error and convert + # it to _DecodeError -- this way we don't have to set up exception- + # handling blocks every time we parse one value. + result = local_unpack('= b'\xF0') + and (double_bytes[0:7] != b'\x00\x00\x00\x00\x00\x00\xF0')): + return (math.nan, new_pos) + + # Note that we expect someone up-stack to catch struct.error and convert + # it to _DecodeError -- this way we don't have to set up exception- + # handling blocks every time we parse one value. + result = local_unpack(' end: + raise _DecodeError('Truncated message.') + while pos < endpoint: + value_start_pos = pos + (element, pos) = _DecodeSignedVarint32(buffer, pos) + # pylint: disable=protected-access + if element in enum_type.values_by_number: + value.append(element) + else: + if not message._unknown_fields: + message._unknown_fields = [] + tag_bytes = encoder.TagBytes(field_number, + wire_format.WIRETYPE_VARINT) + + message._unknown_fields.append( + (tag_bytes, buffer[value_start_pos:pos].tobytes())) + # pylint: enable=protected-access + if pos > endpoint: + if element in enum_type.values_by_number: + del value[-1] # Discard corrupt value. + else: + del message._unknown_fields[-1] + # pylint: enable=protected-access + raise _DecodeError('Packed element was truncated.') + return pos + + return DecodePackedField + elif is_repeated: + tag_bytes = encoder.TagBytes(field_number, wire_format.WIRETYPE_VARINT) + tag_len = len(tag_bytes) + def DecodeRepeatedField( + buffer, pos, end, message, field_dict, current_depth=0 + ): + """Decode serialized repeated enum to its value and a new position. + + Args: + buffer: memoryview of the serialized bytes. + pos: int, position in the memory view to start at. + end: int, end position of serialized data + message: Message object to store unknown fields in + field_dict: Map[Descriptor, Any] to store decoded values in. + + Returns: + int, new position in serialized data. + """ + del current_depth # unused + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + while 1: + (element, new_pos) = _DecodeSignedVarint32(buffer, pos) + # pylint: disable=protected-access + if element in enum_type.values_by_number: + value.append(element) + else: + if not message._unknown_fields: + message._unknown_fields = [] + message._unknown_fields.append( + (tag_bytes, buffer[pos:new_pos].tobytes())) + # pylint: enable=protected-access + # Predict that the next tag is another copy of the same repeated + # field. + pos = new_pos + tag_len + if buffer[new_pos:pos] != tag_bytes or new_pos >= end: + # Prediction failed. Return. + if new_pos > end: + raise _DecodeError('Truncated message.') + return new_pos + + return DecodeRepeatedField + else: + + def DecodeField(buffer, pos, end, message, field_dict, current_depth=0): + """Decode serialized repeated enum to its value and a new position. + + Args: + buffer: memoryview of the serialized bytes. + pos: int, position in the memory view to start at. + end: int, end position of serialized data + message: Message object to store unknown fields in + field_dict: Map[Descriptor, Any] to store decoded values in. + + Returns: + int, new position in serialized data. + """ + del current_depth # unused + value_start_pos = pos + (enum_value, pos) = _DecodeSignedVarint32(buffer, pos) + if pos > end: + raise _DecodeError('Truncated message.') + if clear_if_default and IsDefaultScalarValue(enum_value): + field_dict.pop(key, None) + return pos + # pylint: disable=protected-access + if enum_value in enum_type.values_by_number: + field_dict[key] = enum_value + else: + if not message._unknown_fields: + message._unknown_fields = [] + tag_bytes = encoder.TagBytes(field_number, + wire_format.WIRETYPE_VARINT) + message._unknown_fields.append( + (tag_bytes, buffer[value_start_pos:pos].tobytes())) + # pylint: enable=protected-access + return pos + + return DecodeField + + +# -------------------------------------------------------------------- + + +Int32Decoder = _SimpleDecoder( + wire_format.WIRETYPE_VARINT, _DecodeSignedVarint32) + +Int64Decoder = _SimpleDecoder( + wire_format.WIRETYPE_VARINT, _DecodeSignedVarint) + +UInt32Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint32) +UInt64Decoder = _SimpleDecoder(wire_format.WIRETYPE_VARINT, _DecodeVarint) + +SInt32Decoder = _ModifiedDecoder( + wire_format.WIRETYPE_VARINT, _DecodeVarint32, wire_format.ZigZagDecode) +SInt64Decoder = _ModifiedDecoder( + wire_format.WIRETYPE_VARINT, _DecodeVarint, wire_format.ZigZagDecode) + +# Note that Python conveniently guarantees that when using the '<' prefix on +# formats, they will also have the same size across all platforms (as opposed +# to without the prefix, where their sizes depend on the C compiler's basic +# type sizes). +Fixed32Decoder = _StructPackDecoder(wire_format.WIRETYPE_FIXED32, ' end: + raise _DecodeError('Truncated string.') + value.append(_ConvertToUnicode(buffer[pos:new_pos])) + # Predict that the next tag is another copy of the same repeated field. + pos = new_pos + tag_len + if buffer[new_pos:pos] != tag_bytes or new_pos == end: + # Prediction failed. Return. + return new_pos + + return DecodeRepeatedField + else: + + def DecodeField(buffer, pos, end, message, field_dict, current_depth=0): + del current_depth # unused + (size, pos) = local_DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > end: + raise _DecodeError('Truncated string.') + if clear_if_default and IsDefaultScalarValue(size): + field_dict.pop(key, None) + else: + field_dict[key] = _ConvertToUnicode(buffer[pos:new_pos]) + return new_pos + + return DecodeField + + +def BytesDecoder(field_number, is_repeated, is_packed, key, new_default, + clear_if_default=False): + """Returns a decoder for a bytes field.""" + + local_DecodeVarint = _DecodeVarint + + assert not is_packed + if is_repeated: + tag_bytes = encoder.TagBytes(field_number, + wire_format.WIRETYPE_LENGTH_DELIMITED) + tag_len = len(tag_bytes) + def DecodeRepeatedField( + buffer, pos, end, message, field_dict, current_depth=0 + ): + del current_depth # unused + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + while 1: + (size, pos) = local_DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > end: + raise _DecodeError('Truncated string.') + value.append(buffer[pos:new_pos].tobytes()) + # Predict that the next tag is another copy of the same repeated field. + pos = new_pos + tag_len + if buffer[new_pos:pos] != tag_bytes or new_pos == end: + # Prediction failed. Return. + return new_pos + + return DecodeRepeatedField + else: + + def DecodeField(buffer, pos, end, message, field_dict, current_depth=0): + del current_depth # unused + (size, pos) = local_DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > end: + raise _DecodeError('Truncated string.') + if clear_if_default and IsDefaultScalarValue(size): + field_dict.pop(key, None) + else: + field_dict[key] = buffer[pos:new_pos].tobytes() + return new_pos + + return DecodeField + + +def GroupDecoder(field_number, is_repeated, is_packed, key, new_default): + """Returns a decoder for a group field.""" + + end_tag_bytes = encoder.TagBytes(field_number, + wire_format.WIRETYPE_END_GROUP) + end_tag_len = len(end_tag_bytes) + + assert not is_packed + if is_repeated: + tag_bytes = encoder.TagBytes(field_number, + wire_format.WIRETYPE_START_GROUP) + tag_len = len(tag_bytes) + def DecodeRepeatedField( + buffer, pos, end, message, field_dict, current_depth=0 + ): + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + while 1: + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + # Read sub-message. + current_depth += 1 + if current_depth > _recursion_limit: + raise _DecodeError( + 'Error parsing message: too many levels of nesting.' + ) + pos = value.add()._InternalParse(buffer, pos, end, current_depth) + current_depth -= 1 + # Read end tag. + new_pos = pos+end_tag_len + if buffer[pos:new_pos] != end_tag_bytes or new_pos > end: + raise _DecodeError('Missing group end tag.') + # Predict that the next tag is another copy of the same repeated field. + pos = new_pos + tag_len + if buffer[new_pos:pos] != tag_bytes or new_pos == end: + # Prediction failed. Return. + return new_pos + + return DecodeRepeatedField + else: + + def DecodeField(buffer, pos, end, message, field_dict, current_depth=0): + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + # Read sub-message. + current_depth += 1 + if current_depth > _recursion_limit: + raise _DecodeError('Error parsing message: too many levels of nesting.') + pos = value._InternalParse(buffer, pos, end, current_depth) + current_depth -= 1 + # Read end tag. + new_pos = pos+end_tag_len + if buffer[pos:new_pos] != end_tag_bytes or new_pos > end: + raise _DecodeError('Missing group end tag.') + return new_pos + + return DecodeField + + +def MessageDecoder(field_number, is_repeated, is_packed, key, new_default): + """Returns a decoder for a message field.""" + + local_DecodeVarint = _DecodeVarint + + assert not is_packed + if is_repeated: + tag_bytes = encoder.TagBytes(field_number, + wire_format.WIRETYPE_LENGTH_DELIMITED) + tag_len = len(tag_bytes) + def DecodeRepeatedField( + buffer, pos, end, message, field_dict, current_depth=0 + ): + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + while 1: + # Read length. + (size, pos) = local_DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > end: + raise _DecodeError('Truncated message.') + # Read sub-message. + current_depth += 1 + if current_depth > _recursion_limit: + raise _DecodeError( + 'Error parsing message: too many levels of nesting.' + ) + if ( + value.add()._InternalParse(buffer, pos, new_pos, current_depth) + != new_pos + ): + # The only reason _InternalParse would return early is if it + # encountered an end-group tag. + raise _DecodeError('Unexpected end-group tag.') + current_depth -= 1 + # Predict that the next tag is another copy of the same repeated field. + pos = new_pos + tag_len + if buffer[new_pos:pos] != tag_bytes or new_pos == end: + # Prediction failed. Return. + return new_pos + + return DecodeRepeatedField + else: + + def DecodeField(buffer, pos, end, message, field_dict, current_depth=0): + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + # Read length. + (size, pos) = local_DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > end: + raise _DecodeError('Truncated message.') + # Read sub-message. + current_depth += 1 + if current_depth > _recursion_limit: + raise _DecodeError('Error parsing message: too many levels of nesting.') + if value._InternalParse(buffer, pos, new_pos, current_depth) != new_pos: + # The only reason _InternalParse would return early is if it encountered + # an end-group tag. + raise _DecodeError('Unexpected end-group tag.') + current_depth -= 1 + return new_pos + + return DecodeField + + +# -------------------------------------------------------------------- + +MESSAGE_SET_ITEM_TAG = encoder.TagBytes(1, wire_format.WIRETYPE_START_GROUP) + +def MessageSetItemDecoder(descriptor): + """Returns a decoder for a MessageSet item. + + The parameter is the message Descriptor. + + The message set message looks like this: + message MessageSet { + repeated group Item = 1 { + required int32 type_id = 2; + required string message = 3; + } + } + """ + + type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT) + message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED) + item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP) + + local_ReadTag = ReadTag + local_DecodeVarint = _DecodeVarint + + def DecodeItem(buffer, pos, end, message, field_dict): + """Decode serialized message set to its value and new position. + + Args: + buffer: memoryview of the serialized bytes. + pos: int, position in the memory view to start at. + end: int, end position of serialized data + message: Message object to store unknown fields in + field_dict: Map[Descriptor, Any] to store decoded values in. + + Returns: + int, new position in serialized data. + """ + message_set_item_start = pos + type_id = -1 + message_start = -1 + message_end = -1 + + # Technically, type_id and message can appear in any order, so we need + # a little loop here. + while 1: + (tag_bytes, pos) = local_ReadTag(buffer, pos) + if tag_bytes == type_id_tag_bytes: + (type_id, pos) = local_DecodeVarint(buffer, pos) + elif tag_bytes == message_tag_bytes: + (size, message_start) = local_DecodeVarint(buffer, pos) + pos = message_end = message_start + size + elif tag_bytes == item_end_tag_bytes: + break + else: + field_number, wire_type = DecodeTag(tag_bytes) + _, pos = _DecodeUnknownField(buffer, pos, end, field_number, wire_type) + if pos == -1: + raise _DecodeError('Unexpected end-group tag.') + + if pos > end: + raise _DecodeError('Truncated message.') + + if type_id == -1: + raise _DecodeError('MessageSet item missing type_id.') + if message_start == -1: + raise _DecodeError('MessageSet item missing message.') + + extension = message.Extensions._FindExtensionByNumber(type_id) + # pylint: disable=protected-access + if extension is not None: + value = field_dict.get(extension) + if value is None: + message_type = extension.message_type + if not hasattr(message_type, '_concrete_class'): + message_factory.GetMessageClass(message_type) + value = field_dict.setdefault( + extension, message_type._concrete_class()) + if value._InternalParse(buffer, message_start,message_end) != message_end: + # The only reason _InternalParse would return early is if it encountered + # an end-group tag. + raise _DecodeError('Unexpected end-group tag.') + else: + if not message._unknown_fields: + message._unknown_fields = [] + message._unknown_fields.append( + (MESSAGE_SET_ITEM_TAG, buffer[message_set_item_start:pos].tobytes())) + # pylint: enable=protected-access + + return pos + + return DecodeItem + + +def UnknownMessageSetItemDecoder(): + """Returns a decoder for a Unknown MessageSet item.""" + + type_id_tag_bytes = encoder.TagBytes(2, wire_format.WIRETYPE_VARINT) + message_tag_bytes = encoder.TagBytes(3, wire_format.WIRETYPE_LENGTH_DELIMITED) + item_end_tag_bytes = encoder.TagBytes(1, wire_format.WIRETYPE_END_GROUP) + + def DecodeUnknownItem(buffer): + pos = 0 + end = len(buffer) + message_start = -1 + message_end = -1 + while 1: + (tag_bytes, pos) = ReadTag(buffer, pos) + if tag_bytes == type_id_tag_bytes: + (type_id, pos) = _DecodeVarint(buffer, pos) + elif tag_bytes == message_tag_bytes: + (size, message_start) = _DecodeVarint(buffer, pos) + pos = message_end = message_start + size + elif tag_bytes == item_end_tag_bytes: + break + else: + field_number, wire_type = DecodeTag(tag_bytes) + _, pos = _DecodeUnknownField(buffer, pos, end, field_number, wire_type) + if pos == -1: + raise _DecodeError('Unexpected end-group tag.') + + if pos > end: + raise _DecodeError('Truncated message.') + + if type_id == -1: + raise _DecodeError('MessageSet item missing type_id.') + if message_start == -1: + raise _DecodeError('MessageSet item missing message.') + + return (type_id, buffer[message_start:message_end].tobytes()) + + return DecodeUnknownItem + +# -------------------------------------------------------------------- + +def MapDecoder(field_descriptor, new_default, is_message_map): + """Returns a decoder for a map field.""" + + key = field_descriptor + tag_bytes = encoder.TagBytes(field_descriptor.number, + wire_format.WIRETYPE_LENGTH_DELIMITED) + tag_len = len(tag_bytes) + local_DecodeVarint = _DecodeVarint + # Can't read _concrete_class yet; might not be initialized. + message_type = field_descriptor.message_type + + def DecodeMap(buffer, pos, end, message, field_dict, current_depth=0): + del current_depth # Unused. + submsg = message_type._concrete_class() + value = field_dict.get(key) + if value is None: + value = field_dict.setdefault(key, new_default(message)) + while 1: + # Read length. + (size, pos) = local_DecodeVarint(buffer, pos) + new_pos = pos + size + if new_pos > end: + raise _DecodeError('Truncated message.') + # Read sub-message. + submsg.Clear() + if submsg._InternalParse(buffer, pos, new_pos) != new_pos: + # The only reason _InternalParse would return early is if it + # encountered an end-group tag. + raise _DecodeError('Unexpected end-group tag.') + + if is_message_map: + value[submsg.key].CopyFrom(submsg.value) + else: + value[submsg.key] = submsg.value + + # Predict that the next tag is another copy of the same repeated field. + pos = new_pos + tag_len + if buffer[new_pos:pos] != tag_bytes or new_pos == end: + # Prediction failed. Return. + return new_pos + + return DecodeMap + + +def _DecodeFixed64(buffer, pos): + """Decode a fixed64.""" + new_pos = pos + 8 + return (struct.unpack('= _recursion_limit: + raise _DecodeError('Error parsing message: too many levels of nesting.') + data, pos = _DecodeUnknownFieldSet(buffer, pos, end_pos, current_depth) + current_depth -= 1 + # Check end tag. + if buffer[pos - len(end_tag_bytes) : pos] != end_tag_bytes: + raise _DecodeError('Missing group end tag.') + elif wire_type == wire_format.WIRETYPE_END_GROUP: + return (0, -1) + else: + raise _DecodeError('Wrong wire type in tag.') + + if pos > end_pos: + raise _DecodeError('Truncated message.') + + return (data, pos) diff --git a/lib/python3.12/site-packages/google/protobuf/internal/encoder.py b/lib/python3.12/site-packages/google/protobuf/internal/encoder.py new file mode 100644 index 0000000000000000000000000000000000000000..df7bf1b537f6a3d5f1f4e10009f7e23be91382dd --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/encoder.py @@ -0,0 +1,806 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Code for encoding protocol message primitives. + +Contains the logic for encoding every logical protocol field type +into one of the 5 physical wire types. + +This code is designed to push the Python interpreter's performance to the +limits. + +The basic idea is that at startup time, for every field (i.e. every +FieldDescriptor) we construct two functions: a "sizer" and an "encoder". The +sizer takes a value of this field's type and computes its byte size. The +encoder takes a writer function and a value. It encodes the value into byte +strings and invokes the writer function to write those strings. Typically the +writer function is the write() method of a BytesIO. + +We try to do as much work as possible when constructing the writer and the +sizer rather than when calling them. In particular: +* We copy any needed global functions to local variables, so that we do not need + to do costly global table lookups at runtime. +* Similarly, we try to do any attribute lookups at startup time if possible. +* Every field's tag is encoded to bytes at startup, since it can't change at + runtime. +* Whatever component of the field size we can compute at startup, we do. +* We *avoid* sharing code if doing so would make the code slower and not sharing + does not burden us too much. For example, encoders for repeated fields do + not just call the encoders for singular fields in a loop because this would + add an extra function call overhead for every loop iteration; instead, we + manually inline the single-value encoder into the loop. +* If a Python function lacks a return statement, Python actually generates + instructions to pop the result of the last statement off the stack, push + None onto the stack, and then return that. If we really don't care what + value is returned, then we can save two instructions by returning the + result of the last statement. It looks funny but it helps. +* We assume that type and bounds checking has happened at a higher level. +""" + +__author__ = 'kenton@google.com (Kenton Varda)' + +import struct + +from google.protobuf.internal import wire_format + + +# This will overflow and thus become IEEE-754 "infinity". We would use +# "float('inf')" but it doesn't work on Windows pre-Python-2.6. +_POS_INF = 1e10000 +_NEG_INF = -_POS_INF + + +def _VarintSize(value): + """Compute the size of a varint value.""" + if value <= 0x7f: return 1 + if value <= 0x3fff: return 2 + if value <= 0x1fffff: return 3 + if value <= 0xfffffff: return 4 + if value <= 0x7ffffffff: return 5 + if value <= 0x3ffffffffff: return 6 + if value <= 0x1ffffffffffff: return 7 + if value <= 0xffffffffffffff: return 8 + if value <= 0x7fffffffffffffff: return 9 + return 10 + + +def _SignedVarintSize(value): + """Compute the size of a signed varint value.""" + if value < 0: return 10 + if value <= 0x7f: return 1 + if value <= 0x3fff: return 2 + if value <= 0x1fffff: return 3 + if value <= 0xfffffff: return 4 + if value <= 0x7ffffffff: return 5 + if value <= 0x3ffffffffff: return 6 + if value <= 0x1ffffffffffff: return 7 + if value <= 0xffffffffffffff: return 8 + if value <= 0x7fffffffffffffff: return 9 + return 10 + + +def _TagSize(field_number): + """Returns the number of bytes required to serialize a tag with this field + number.""" + # Just pass in type 0, since the type won't affect the tag+type size. + return _VarintSize(wire_format.PackTag(field_number, 0)) + + +# -------------------------------------------------------------------- +# In this section we define some generic sizers. Each of these functions +# takes parameters specific to a particular field type, e.g. int32 or fixed64. +# It returns another function which in turn takes parameters specific to a +# particular field, e.g. the field number and whether it is repeated or packed. +# Look at the next section to see how these are used. + + +def _SimpleSizer(compute_value_size): + """A sizer which uses the function compute_value_size to compute the size of + each value. Typically compute_value_size is _VarintSize.""" + + def SpecificSizer(field_number, is_repeated, is_packed): + tag_size = _TagSize(field_number) + if is_packed: + local_VarintSize = _VarintSize + def PackedFieldSize(value): + result = 0 + for element in value: + result += compute_value_size(element) + return result + local_VarintSize(result) + tag_size + return PackedFieldSize + elif is_repeated: + def RepeatedFieldSize(value): + result = tag_size * len(value) + for element in value: + result += compute_value_size(element) + return result + return RepeatedFieldSize + else: + def FieldSize(value): + return tag_size + compute_value_size(value) + return FieldSize + + return SpecificSizer + + +def _ModifiedSizer(compute_value_size, modify_value): + """Like SimpleSizer, but modify_value is invoked on each value before it is + passed to compute_value_size. modify_value is typically ZigZagEncode.""" + + def SpecificSizer(field_number, is_repeated, is_packed): + tag_size = _TagSize(field_number) + if is_packed: + local_VarintSize = _VarintSize + def PackedFieldSize(value): + result = 0 + for element in value: + result += compute_value_size(modify_value(element)) + return result + local_VarintSize(result) + tag_size + return PackedFieldSize + elif is_repeated: + def RepeatedFieldSize(value): + result = tag_size * len(value) + for element in value: + result += compute_value_size(modify_value(element)) + return result + return RepeatedFieldSize + else: + def FieldSize(value): + return tag_size + compute_value_size(modify_value(value)) + return FieldSize + + return SpecificSizer + + +def _FixedSizer(value_size): + """Like _SimpleSizer except for a fixed-size field. The input is the size + of one value.""" + + def SpecificSizer(field_number, is_repeated, is_packed): + tag_size = _TagSize(field_number) + if is_packed: + local_VarintSize = _VarintSize + def PackedFieldSize(value): + result = len(value) * value_size + return result + local_VarintSize(result) + tag_size + return PackedFieldSize + elif is_repeated: + element_size = value_size + tag_size + def RepeatedFieldSize(value): + return len(value) * element_size + return RepeatedFieldSize + else: + field_size = value_size + tag_size + def FieldSize(value): + return field_size + return FieldSize + + return SpecificSizer + + +# ==================================================================== +# Here we declare a sizer constructor for each field type. Each "sizer +# constructor" is a function that takes (field_number, is_repeated, is_packed) +# as parameters and returns a sizer, which in turn takes a field value as +# a parameter and returns its encoded size. + + +Int32Sizer = Int64Sizer = EnumSizer = _SimpleSizer(_SignedVarintSize) + +UInt32Sizer = UInt64Sizer = _SimpleSizer(_VarintSize) + +SInt32Sizer = SInt64Sizer = _ModifiedSizer( + _SignedVarintSize, wire_format.ZigZagEncode) + +Fixed32Sizer = SFixed32Sizer = FloatSizer = _FixedSizer(4) +Fixed64Sizer = SFixed64Sizer = DoubleSizer = _FixedSizer(8) + +BoolSizer = _FixedSizer(1) + + +def StringSizer(field_number, is_repeated, is_packed): + """Returns a sizer for a string field.""" + + tag_size = _TagSize(field_number) + local_VarintSize = _VarintSize + local_len = len + assert not is_packed + if is_repeated: + def RepeatedFieldSize(value): + result = tag_size * len(value) + for element in value: + l = local_len(element.encode('utf-8')) + result += local_VarintSize(l) + l + return result + return RepeatedFieldSize + else: + def FieldSize(value): + l = local_len(value.encode('utf-8')) + return tag_size + local_VarintSize(l) + l + return FieldSize + + +def BytesSizer(field_number, is_repeated, is_packed): + """Returns a sizer for a bytes field.""" + + tag_size = _TagSize(field_number) + local_VarintSize = _VarintSize + local_len = len + assert not is_packed + if is_repeated: + def RepeatedFieldSize(value): + result = tag_size * len(value) + for element in value: + l = local_len(element) + result += local_VarintSize(l) + l + return result + return RepeatedFieldSize + else: + def FieldSize(value): + l = local_len(value) + return tag_size + local_VarintSize(l) + l + return FieldSize + + +def GroupSizer(field_number, is_repeated, is_packed): + """Returns a sizer for a group field.""" + + tag_size = _TagSize(field_number) * 2 + assert not is_packed + if is_repeated: + def RepeatedFieldSize(value): + result = tag_size * len(value) + for element in value: + result += element.ByteSize() + return result + return RepeatedFieldSize + else: + def FieldSize(value): + return tag_size + value.ByteSize() + return FieldSize + + +def MessageSizer(field_number, is_repeated, is_packed): + """Returns a sizer for a message field.""" + + tag_size = _TagSize(field_number) + local_VarintSize = _VarintSize + assert not is_packed + if is_repeated: + def RepeatedFieldSize(value): + result = tag_size * len(value) + for element in value: + l = element.ByteSize() + result += local_VarintSize(l) + l + return result + return RepeatedFieldSize + else: + def FieldSize(value): + l = value.ByteSize() + return tag_size + local_VarintSize(l) + l + return FieldSize + + +# -------------------------------------------------------------------- +# MessageSet is special: it needs custom logic to compute its size properly. + + +def MessageSetItemSizer(field_number): + """Returns a sizer for extensions of MessageSet. + + The message set message looks like this: + message MessageSet { + repeated group Item = 1 { + required int32 type_id = 2; + required string message = 3; + } + } + """ + static_size = (_TagSize(1) * 2 + _TagSize(2) + _VarintSize(field_number) + + _TagSize(3)) + local_VarintSize = _VarintSize + + def FieldSize(value): + l = value.ByteSize() + return static_size + local_VarintSize(l) + l + + return FieldSize + + +# -------------------------------------------------------------------- +# Map is special: it needs custom logic to compute its size properly. + + +def MapSizer(field_descriptor, is_message_map): + """Returns a sizer for a map field.""" + + # Can't look at field_descriptor.message_type._concrete_class because it may + # not have been initialized yet. + message_type = field_descriptor.message_type + message_sizer = MessageSizer(field_descriptor.number, False, False) + + def FieldSize(map_value): + total = 0 + for key in map_value: + value = map_value[key] + # It's wasteful to create the messages and throw them away one second + # later since we'll do the same for the actual encode. But there's not an + # obvious way to avoid this within the current design without tons of code + # duplication. For message map, value.ByteSize() should be called to + # update the status. + entry_msg = message_type._concrete_class(key=key, value=value) + total += message_sizer(entry_msg) + if is_message_map: + value.ByteSize() + return total + + return FieldSize + +# ==================================================================== +# Encoders! + + +def _VarintEncoder(): + """Return an encoder for a basic varint value (does not include tag).""" + + local_int2byte = struct.Struct('>B').pack + + def EncodeVarint(write, value, unused_deterministic=None): + bits = value & 0x7f + value >>= 7 + while value: + write(local_int2byte(0x80|bits)) + bits = value & 0x7f + value >>= 7 + return write(local_int2byte(bits)) + + return EncodeVarint + + +def _SignedVarintEncoder(): + """Return an encoder for a basic signed varint value (does not include + tag).""" + + local_int2byte = struct.Struct('>B').pack + + def EncodeSignedVarint(write, value, unused_deterministic=None): + if value < 0: + value += (1 << 64) + bits = value & 0x7f + value >>= 7 + while value: + write(local_int2byte(0x80|bits)) + bits = value & 0x7f + value >>= 7 + return write(local_int2byte(bits)) + + return EncodeSignedVarint + + +_EncodeVarint = _VarintEncoder() +_EncodeSignedVarint = _SignedVarintEncoder() + + +def _VarintBytes(value): + """Encode the given integer as a varint and return the bytes. This is only + called at startup time so it doesn't need to be fast.""" + + pieces = [] + _EncodeVarint(pieces.append, value, True) + return b"".join(pieces) + + +def TagBytes(field_number, wire_type): + """Encode the given tag and return the bytes. Only called at startup.""" + + return bytes(_VarintBytes(wire_format.PackTag(field_number, wire_type))) + +# -------------------------------------------------------------------- +# As with sizers (see above), we have a number of common encoder +# implementations. + + +def _SimpleEncoder(wire_type, encode_value, compute_value_size): + """Return a constructor for an encoder for fields of a particular type. + + Args: + wire_type: The field's wire type, for encoding tags. + encode_value: A function which encodes an individual value, e.g. + _EncodeVarint(). + compute_value_size: A function which computes the size of an individual + value, e.g. _VarintSize(). + """ + + def SpecificEncoder(field_number, is_repeated, is_packed): + if is_packed: + tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local_EncodeVarint = _EncodeVarint + def EncodePackedField(write, value, deterministic): + write(tag_bytes) + size = 0 + for element in value: + size += compute_value_size(element) + local_EncodeVarint(write, size, deterministic) + for element in value: + encode_value(write, element, deterministic) + return EncodePackedField + elif is_repeated: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeRepeatedField(write, value, deterministic): + for element in value: + write(tag_bytes) + encode_value(write, element, deterministic) + return EncodeRepeatedField + else: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeField(write, value, deterministic): + write(tag_bytes) + return encode_value(write, value, deterministic) + return EncodeField + + return SpecificEncoder + + +def _ModifiedEncoder(wire_type, encode_value, compute_value_size, modify_value): + """Like SimpleEncoder but additionally invokes modify_value on every value + before passing it to encode_value. Usually modify_value is ZigZagEncode.""" + + def SpecificEncoder(field_number, is_repeated, is_packed): + if is_packed: + tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local_EncodeVarint = _EncodeVarint + def EncodePackedField(write, value, deterministic): + write(tag_bytes) + size = 0 + for element in value: + size += compute_value_size(modify_value(element)) + local_EncodeVarint(write, size, deterministic) + for element in value: + encode_value(write, modify_value(element), deterministic) + return EncodePackedField + elif is_repeated: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeRepeatedField(write, value, deterministic): + for element in value: + write(tag_bytes) + encode_value(write, modify_value(element), deterministic) + return EncodeRepeatedField + else: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeField(write, value, deterministic): + write(tag_bytes) + return encode_value(write, modify_value(value), deterministic) + return EncodeField + + return SpecificEncoder + + +def _StructPackEncoder(wire_type, format): + """Return a constructor for an encoder for a fixed-width field. + + Args: + wire_type: The field's wire type, for encoding tags. + format: The format string to pass to struct.pack(). + """ + + value_size = struct.calcsize(format) + + def SpecificEncoder(field_number, is_repeated, is_packed): + local_struct_pack = struct.pack + if is_packed: + tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local_EncodeVarint = _EncodeVarint + def EncodePackedField(write, value, deterministic): + write(tag_bytes) + local_EncodeVarint(write, len(value) * value_size, deterministic) + for element in value: + write(local_struct_pack(format, element)) + return EncodePackedField + elif is_repeated: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeRepeatedField(write, value, unused_deterministic=None): + for element in value: + write(tag_bytes) + write(local_struct_pack(format, element)) + return EncodeRepeatedField + else: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeField(write, value, unused_deterministic=None): + write(tag_bytes) + return write(local_struct_pack(format, value)) + return EncodeField + + return SpecificEncoder + + +def _FloatingPointEncoder(wire_type, format): + """Return a constructor for an encoder for float fields. + + This is like StructPackEncoder, but catches errors that may be due to + passing non-finite floating-point values to struct.pack, and makes a + second attempt to encode those values. + + Args: + wire_type: The field's wire type, for encoding tags. + format: The format string to pass to struct.pack(). + """ + + value_size = struct.calcsize(format) + if value_size == 4: + def EncodeNonFiniteOrRaise(write, value): + # Remember that the serialized form uses little-endian byte order. + if value == _POS_INF: + write(b'\x00\x00\x80\x7F') + elif value == _NEG_INF: + write(b'\x00\x00\x80\xFF') + elif value != value: # NaN + write(b'\x00\x00\xC0\x7F') + else: + raise + elif value_size == 8: + def EncodeNonFiniteOrRaise(write, value): + if value == _POS_INF: + write(b'\x00\x00\x00\x00\x00\x00\xF0\x7F') + elif value == _NEG_INF: + write(b'\x00\x00\x00\x00\x00\x00\xF0\xFF') + elif value != value: # NaN + write(b'\x00\x00\x00\x00\x00\x00\xF8\x7F') + else: + raise + else: + raise ValueError('Can\'t encode floating-point values that are ' + '%d bytes long (only 4 or 8)' % value_size) + + def SpecificEncoder(field_number, is_repeated, is_packed): + local_struct_pack = struct.pack + if is_packed: + tag_bytes = TagBytes(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED) + local_EncodeVarint = _EncodeVarint + def EncodePackedField(write, value, deterministic): + write(tag_bytes) + local_EncodeVarint(write, len(value) * value_size, deterministic) + for element in value: + # This try/except block is going to be faster than any code that + # we could write to check whether element is finite. + try: + write(local_struct_pack(format, element)) + except SystemError: + EncodeNonFiniteOrRaise(write, element) + return EncodePackedField + elif is_repeated: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeRepeatedField(write, value, unused_deterministic=None): + for element in value: + write(tag_bytes) + try: + write(local_struct_pack(format, element)) + except SystemError: + EncodeNonFiniteOrRaise(write, element) + return EncodeRepeatedField + else: + tag_bytes = TagBytes(field_number, wire_type) + def EncodeField(write, value, unused_deterministic=None): + write(tag_bytes) + try: + write(local_struct_pack(format, value)) + except SystemError: + EncodeNonFiniteOrRaise(write, value) + return EncodeField + + return SpecificEncoder + + +# ==================================================================== +# Here we declare an encoder constructor for each field type. These work +# very similarly to sizer constructors, described earlier. + + +Int32Encoder = Int64Encoder = EnumEncoder = _SimpleEncoder( + wire_format.WIRETYPE_VARINT, _EncodeSignedVarint, _SignedVarintSize) + +UInt32Encoder = UInt64Encoder = _SimpleEncoder( + wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize) + +SInt32Encoder = SInt64Encoder = _ModifiedEncoder( + wire_format.WIRETYPE_VARINT, _EncodeVarint, _VarintSize, + wire_format.ZigZagEncode) + +# Note that Python conveniently guarantees that when using the '<' prefix on +# formats, they will also have the same size across all platforms (as opposed +# to without the prefix, where their sizes depend on the C compiler's basic +# type sizes). +Fixed32Encoder = _StructPackEncoder(wire_format.WIRETYPE_FIXED32, ' str + ValueType = int + + def __init__(self, enum_type): + """Inits EnumTypeWrapper with an EnumDescriptor.""" + self._enum_type = enum_type + self.DESCRIPTOR = enum_type # pylint: disable=invalid-name + + def Name(self, number): # pylint: disable=invalid-name + """Returns a string containing the name of an enum value.""" + try: + return self._enum_type.values_by_number[number].name + except KeyError: + pass # fall out to break exception chaining + + if not isinstance(number, int): + raise TypeError( + 'Enum value for {} must be an int, but got {} {!r}.'.format( + self._enum_type.name, type(number), number)) + else: + # repr here to handle the odd case when you pass in a boolean. + raise ValueError('Enum {} has no name defined for value {!r}'.format( + self._enum_type.name, number)) + + def Value(self, name): # pylint: disable=invalid-name + """Returns the value corresponding to the given enum name.""" + try: + return self._enum_type.values_by_name[name].number + except KeyError: + pass # fall out to break exception chaining + raise ValueError('Enum {} has no value defined for name {!r}'.format( + self._enum_type.name, name)) + + def keys(self): + """Return a list of the string names in the enum. + + Returns: + A list of strs, in the order they were defined in the .proto file. + """ + + return [value_descriptor.name + for value_descriptor in self._enum_type.values] + + def values(self): + """Return a list of the integer values in the enum. + + Returns: + A list of ints, in the order they were defined in the .proto file. + """ + + return [value_descriptor.number + for value_descriptor in self._enum_type.values] + + def items(self): + """Return a list of the (name, value) pairs of the enum. + + Returns: + A list of (str, int) pairs, in the order they were defined + in the .proto file. + """ + return [(value_descriptor.name, value_descriptor.number) + for value_descriptor in self._enum_type.values] + + def __getattr__(self, name): + """Returns the value corresponding to the given enum name.""" + try: + return super( + EnumTypeWrapper, + self).__getattribute__('_enum_type').values_by_name[name].number + except KeyError: + pass # fall out to break exception chaining + raise AttributeError('Enum {} has no value defined for name {!r}'.format( + self._enum_type.name, name)) + + def __or__(self, other): + """Returns the union type of self and other.""" + if sys.version_info >= (3, 10): + return type(self) | other + else: + raise NotImplementedError( + 'You may not use | on EnumTypes (or classes) below python 3.10' + ) diff --git a/lib/python3.12/site-packages/google/protobuf/internal/extension_dict.py b/lib/python3.12/site-packages/google/protobuf/internal/extension_dict.py new file mode 100644 index 0000000000000000000000000000000000000000..8163ab94b1bd2a1e3a61ff3dc63e608cf315c105 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/extension_dict.py @@ -0,0 +1,194 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains _ExtensionDict class to represent extensions. +""" + +from google.protobuf.internal import type_checkers +from google.protobuf.descriptor import FieldDescriptor + + +def _VerifyExtensionHandle(message, extension_handle): + """Verify that the given extension handle is valid.""" + + if not isinstance(extension_handle, FieldDescriptor): + raise KeyError('HasExtension() expects an extension handle, got: %s' % + extension_handle) + + if not extension_handle.is_extension: + raise KeyError('"%s" is not an extension.' % extension_handle.full_name) + + if not extension_handle.containing_type: + raise KeyError('"%s" is missing a containing_type.' + % extension_handle.full_name) + + if extension_handle.containing_type is not message.DESCRIPTOR: + raise KeyError('Extension "%s" extends message type "%s", but this ' + 'message is of type "%s".' % + (extension_handle.full_name, + extension_handle.containing_type.full_name, + message.DESCRIPTOR.full_name)) + + +# TODO: Unify error handling of "unknown extension" crap. +# TODO: Support iteritems()-style iteration over all +# extensions with the "has" bits turned on? +class _ExtensionDict(object): + + """Dict-like container for Extension fields on proto instances. + + Note that in all cases we expect extension handles to be + FieldDescriptors. + """ + + def __init__(self, extended_message): + """ + Args: + extended_message: Message instance for which we are the Extensions dict. + """ + self._extended_message = extended_message + + def __getitem__(self, extension_handle): + """Returns the current value of the given extension handle.""" + + _VerifyExtensionHandle(self._extended_message, extension_handle) + + result = self._extended_message._fields.get(extension_handle) + if result is not None: + return result + + if extension_handle.is_repeated: + result = extension_handle._default_constructor(self._extended_message) + elif extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: + message_type = extension_handle.message_type + if not hasattr(message_type, '_concrete_class'): + # pylint: disable=g-import-not-at-top + from google.protobuf import message_factory + message_factory.GetMessageClass(message_type) + if not hasattr(extension_handle.message_type, '_concrete_class'): + from google.protobuf import message_factory + message_factory.GetMessageClass(extension_handle.message_type) + result = extension_handle.message_type._concrete_class() + try: + result._SetListener(self._extended_message._listener_for_children) + except ReferenceError: + pass + else: + # Singular scalar -- just return the default without inserting into the + # dict. + return extension_handle.default_value + + # Atomically check if another thread has preempted us and, if not, swap + # in the new object we just created. If someone has preempted us, we + # take that object and discard ours. + # WARNING: We are relying on setdefault() being atomic. This is true + # in CPython but we haven't investigated others. This warning appears + # in several other locations in this file. + result = self._extended_message._fields.setdefault( + extension_handle, result) + + return result + + def __eq__(self, other): + if not isinstance(other, self.__class__): + return False + + my_fields = self._extended_message.ListFields() + other_fields = other._extended_message.ListFields() + + # Get rid of non-extension fields. + my_fields = [field for field in my_fields if field.is_extension] + other_fields = [field for field in other_fields if field.is_extension] + + return my_fields == other_fields + + def __ne__(self, other): + return not self == other + + def __len__(self): + fields = self._extended_message.ListFields() + # Get rid of non-extension fields. + extension_fields = [field for field in fields if field[0].is_extension] + return len(extension_fields) + + def __hash__(self): + raise TypeError('unhashable object') + + # Note that this is only meaningful for non-repeated, scalar extension + # fields. Note also that we may have to call _Modified() when we do + # successfully set a field this way, to set any necessary "has" bits in the + # ancestors of the extended message. + def __setitem__(self, extension_handle, value): + """If extension_handle specifies a non-repeated, scalar extension + field, sets the value of that field. + """ + + _VerifyExtensionHandle(self._extended_message, extension_handle) + + if (extension_handle.is_repeated or + extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE): + raise TypeError( + 'Cannot assign to extension "%s" because it is a repeated or ' + 'composite type.' % extension_handle.full_name) + + # It's slightly wasteful to lookup the type checker each time, + # but we expect this to be a vanishingly uncommon case anyway. + type_checker = type_checkers.GetTypeChecker(extension_handle) + # pylint: disable=protected-access + self._extended_message._fields[extension_handle] = ( + type_checker.CheckValue(value)) + self._extended_message._Modified() + + def __delitem__(self, extension_handle): + self._extended_message.ClearExtension(extension_handle) + + def _FindExtensionByName(self, name): + """Tries to find a known extension with the specified name. + + Args: + name: Extension full name. + + Returns: + Extension field descriptor. + """ + descriptor = self._extended_message.DESCRIPTOR + extensions = descriptor.file.pool._extensions_by_name[descriptor] + return extensions.get(name, None) + + def _FindExtensionByNumber(self, number): + """Tries to find a known extension with the field number. + + Args: + number: Extension field number. + + Returns: + Extension field descriptor. + """ + descriptor = self._extended_message.DESCRIPTOR + extensions = descriptor.file.pool._extensions_by_number[descriptor] + return extensions.get(number, None) + + def __iter__(self): + # Return a generator over the populated extension fields + return (f[0] for f in self._extended_message.ListFields() + if f[0].is_extension) + + def __contains__(self, extension_handle): + _VerifyExtensionHandle(self._extended_message, extension_handle) + + if extension_handle not in self._extended_message._fields: + return False + + if extension_handle.is_repeated: + return bool(self._extended_message._fields.get(extension_handle)) + + if extension_handle.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: + value = self._extended_message._fields.get(extension_handle) + # pylint: disable=protected-access + return value is not None and value._is_present_in_parent + + return True diff --git a/lib/python3.12/site-packages/google/protobuf/internal/field_mask.py b/lib/python3.12/site-packages/google/protobuf/internal/field_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..0357486e9fd1b5e7dbdb79c59dda33eaa0a452dc --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/field_mask.py @@ -0,0 +1,312 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains FieldMask class.""" + +from google.protobuf.descriptor import FieldDescriptor + + +class FieldMask(object): + """Class for FieldMask message type.""" + + __slots__ = () + + def ToJsonString(self): + """Converts FieldMask to string according to proto3 JSON spec.""" + camelcase_paths = [] + for path in self.paths: + camelcase_paths.append(_SnakeCaseToCamelCase(path)) + return ','.join(camelcase_paths) + + def FromJsonString(self, value): + """Converts string to FieldMask according to proto3 JSON spec.""" + if not isinstance(value, str): + raise ValueError('FieldMask JSON value not a string: {!r}'.format(value)) + self.Clear() + if value: + for path in value.split(','): + self.paths.append(_CamelCaseToSnakeCase(path)) + + def IsValidForDescriptor(self, message_descriptor): + """Checks whether the FieldMask is valid for Message Descriptor.""" + for path in self.paths: + if not _IsValidPath(message_descriptor, path): + return False + return True + + def AllFieldsFromDescriptor(self, message_descriptor): + """Gets all direct fields of Message Descriptor to FieldMask.""" + self.Clear() + for field in message_descriptor.fields: + self.paths.append(field.name) + + def CanonicalFormFromMask(self, mask): + """Converts a FieldMask to the canonical form. + + Removes paths that are covered by another path. For example, + "foo.bar" is covered by "foo" and will be removed if "foo" + is also in the FieldMask. Then sorts all paths in alphabetical order. + + Args: + mask: The original FieldMask to be converted. + """ + tree = _FieldMaskTree(mask) + tree.ToFieldMask(self) + + def Union(self, mask1, mask2): + """Merges mask1 and mask2 into this FieldMask.""" + _CheckFieldMaskMessage(mask1) + _CheckFieldMaskMessage(mask2) + tree = _FieldMaskTree(mask1) + tree.MergeFromFieldMask(mask2) + tree.ToFieldMask(self) + + def Intersect(self, mask1, mask2): + """Intersects mask1 and mask2 into this FieldMask.""" + _CheckFieldMaskMessage(mask1) + _CheckFieldMaskMessage(mask2) + tree = _FieldMaskTree(mask1) + intersection = _FieldMaskTree() + for path in mask2.paths: + tree.IntersectPath(path, intersection) + intersection.ToFieldMask(self) + + def MergeMessage( + self, source, destination, + replace_message_field=False, replace_repeated_field=False): + """Merges fields specified in FieldMask from source to destination. + + Args: + source: Source message. + destination: The destination message to be merged into. + replace_message_field: Replace message field if True. Merge message + field if False. + replace_repeated_field: Replace repeated field if True. Append + elements of repeated field if False. + """ + tree = _FieldMaskTree(self) + tree.MergeMessage( + source, destination, replace_message_field, replace_repeated_field) + + +def _IsValidPath(message_descriptor, path): + """Checks whether the path is valid for Message Descriptor.""" + parts = path.split('.') + last = parts.pop() + for name in parts: + field = message_descriptor.fields_by_name.get(name) + if (field is None or + field.is_repeated or + field.type != FieldDescriptor.TYPE_MESSAGE): + return False + message_descriptor = field.message_type + return last in message_descriptor.fields_by_name + + +def _CheckFieldMaskMessage(message): + """Raises ValueError if message is not a FieldMask.""" + message_descriptor = message.DESCRIPTOR + if (message_descriptor.name != 'FieldMask' or + message_descriptor.file.name != 'google/protobuf/field_mask.proto'): + raise ValueError('Message {0} is not a FieldMask.'.format( + message_descriptor.full_name)) + + +def _SnakeCaseToCamelCase(path_name): + """Converts a path name from snake_case to camelCase.""" + result = [] + after_underscore = False + for c in path_name: + if c.isupper(): + raise ValueError( + 'Fail to print FieldMask to Json string: Path name ' + '{0} must not contain uppercase letters.'.format(path_name)) + if after_underscore: + if c.islower(): + result.append(c.upper()) + after_underscore = False + else: + raise ValueError( + 'Fail to print FieldMask to Json string: The ' + 'character after a "_" must be a lowercase letter ' + 'in path name {0}.'.format(path_name)) + elif c == '_': + after_underscore = True + else: + result += c + + if after_underscore: + raise ValueError('Fail to print FieldMask to Json string: Trailing "_" ' + 'in path name {0}.'.format(path_name)) + return ''.join(result) + + +def _CamelCaseToSnakeCase(path_name): + """Converts a field name from camelCase to snake_case.""" + result = [] + for c in path_name: + if c == '_': + raise ValueError('Fail to parse FieldMask: Path name ' + '{0} must not contain "_"s.'.format(path_name)) + if c.isupper(): + result += '_' + result += c.lower() + else: + result += c + return ''.join(result) + + +class _FieldMaskTree(object): + """Represents a FieldMask in a tree structure. + + For example, given a FieldMask "foo.bar,foo.baz,bar.baz", + the FieldMaskTree will be: + [_root] -+- foo -+- bar + | | + | +- baz + | + +- bar --- baz + In the tree, each leaf node represents a field path. + """ + + __slots__ = ('_root',) + + def __init__(self, field_mask=None): + """Initializes the tree by FieldMask.""" + self._root = {} + if field_mask: + self.MergeFromFieldMask(field_mask) + + def MergeFromFieldMask(self, field_mask): + """Merges a FieldMask to the tree.""" + for path in field_mask.paths: + self.AddPath(path) + + def AddPath(self, path): + """Adds a field path into the tree. + + If the field path to add is a sub-path of an existing field path + in the tree (i.e., a leaf node), it means the tree already matches + the given path so nothing will be added to the tree. If the path + matches an existing non-leaf node in the tree, that non-leaf node + will be turned into a leaf node with all its children removed because + the path matches all the node's children. Otherwise, a new path will + be added. + + Args: + path: The field path to add. + """ + node = self._root + for name in path.split('.'): + if name not in node: + node[name] = {} + elif not node[name]: + # Pre-existing empty node implies we already have this entire tree. + return + node = node[name] + # Remove any sub-trees we might have had. + node.clear() + + def ToFieldMask(self, field_mask): + """Converts the tree to a FieldMask.""" + field_mask.Clear() + _AddFieldPaths(self._root, '', field_mask) + + def IntersectPath(self, path, intersection): + """Calculates the intersection part of a field path with this tree. + + Args: + path: The field path to calculates. + intersection: The out tree to record the intersection part. + """ + node = self._root + for name in path.split('.'): + if name not in node: + return + elif not node[name]: + intersection.AddPath(path) + return + node = node[name] + intersection.AddLeafNodes(path, node) + + def AddLeafNodes(self, prefix, node): + """Adds leaf nodes begin with prefix to this tree.""" + if not node: + self.AddPath(prefix) + for name in node: + child_path = prefix + '.' + name + self.AddLeafNodes(child_path, node[name]) + + def MergeMessage( + self, source, destination, + replace_message, replace_repeated): + """Merge all fields specified by this tree from source to destination.""" + _MergeMessage( + self._root, source, destination, replace_message, replace_repeated) + + +def _StrConvert(value): + """Converts value to str if it is not.""" + # This file is imported by c extension and some methods like ClearField + # requires string for the field name. py2/py3 has different text + # type and may use unicode. + if not isinstance(value, str): + return value.encode('utf-8') + return value + + +def _MergeMessage( + node, source, destination, replace_message, replace_repeated): + """Merge all fields specified by a sub-tree from source to destination.""" + source_descriptor = source.DESCRIPTOR + for name in node: + child = node[name] + field = source_descriptor.fields_by_name[name] + if field is None: + raise ValueError('Error: Can\'t find field {0} in message {1}.'.format( + name, source_descriptor.full_name)) + if child: + # Sub-paths are only allowed for singular message fields. + if (field.is_repeated or + field.cpp_type != FieldDescriptor.CPPTYPE_MESSAGE): + raise ValueError('Error: Field {0} in message {1} is not a singular ' + 'message field and cannot have sub-fields.'.format( + name, source_descriptor.full_name)) + if source.HasField(name): + _MergeMessage( + child, getattr(source, name), getattr(destination, name), + replace_message, replace_repeated) + continue + if field.is_repeated: + if replace_repeated: + destination.ClearField(_StrConvert(name)) + repeated_source = getattr(source, name) + repeated_destination = getattr(destination, name) + repeated_destination.MergeFrom(repeated_source) + else: + if field.cpp_type == FieldDescriptor.CPPTYPE_MESSAGE: + if replace_message: + destination.ClearField(_StrConvert(name)) + if source.HasField(name): + getattr(destination, name).MergeFrom(getattr(source, name)) + elif not field.has_presence or source.HasField(name): + setattr(destination, name, getattr(source, name)) + else: + destination.ClearField(_StrConvert(name)) + + +def _AddFieldPaths(node, prefix, field_mask): + """Adds the field paths descended from node to field_mask.""" + if not node and prefix: + field_mask.paths.append(prefix) + return + for name in sorted(node): + if prefix: + child_path = prefix + '.' + name + else: + child_path = name + _AddFieldPaths(node[name], child_path, field_mask) diff --git a/lib/python3.12/site-packages/google/protobuf/internal/message_listener.py b/lib/python3.12/site-packages/google/protobuf/internal/message_listener.py new file mode 100644 index 0000000000000000000000000000000000000000..ff1c127d549dcd8973fdbe67fda09b9ce3460abe --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/message_listener.py @@ -0,0 +1,55 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Defines a listener interface for observing certain +state transitions on Message objects. + +Also defines a null implementation of this interface. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + + +class MessageListener(object): + + """Listens for modifications made to a message. Meant to be registered via + Message._SetListener(). + + Attributes: + dirty: If True, then calling Modified() would be a no-op. This can be + used to avoid these calls entirely in the common case. + """ + + def Modified(self): + """Called every time the message is modified in such a way that the parent + message may need to be updated. This currently means either: + (a) The message was modified for the first time, so the parent message + should henceforth mark the message as present. + (b) The message's cached byte size became dirty -- i.e. the message was + modified for the first time after a previous call to ByteSize(). + Therefore the parent should also mark its byte size as dirty. + Note that (a) implies (b), since new objects start out with a client cached + size (zero). However, we document (a) explicitly because it is important. + + Modified() will *only* be called in response to one of these two events -- + not every time the sub-message is modified. + + Note that if the listener's |dirty| attribute is true, then calling + Modified at the moment would be a no-op, so it can be skipped. Performance- + sensitive callers should check this attribute directly before calling since + it will be true most of the time. + """ + + raise NotImplementedError + + +class NullMessageListener(object): + + """No-op MessageListener implementation.""" + + def Modified(self): + pass diff --git a/lib/python3.12/site-packages/google/protobuf/internal/python_edition_defaults.py b/lib/python3.12/site-packages/google/protobuf/internal/python_edition_defaults.py new file mode 100644 index 0000000000000000000000000000000000000000..97d2be08f3c15464c4b3ea90a080a90b3b25cf31 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/python_edition_defaults.py @@ -0,0 +1,5 @@ +""" +This file contains the serialized FeatureSetDefaults object corresponding to +the Pure Python runtime. This is used for feature resolution under Editions. +""" +_PROTOBUF_INTERNAL_PYTHON_EDITION_DEFAULTS = b"\n\027\030\204\007\"\000*\020\010\001\020\002\030\002 \003(\0010\0028\002@\001\n\027\030\347\007\"\000*\020\010\002\020\001\030\001 \002(\0010\0018\002@\001\n\027\030\350\007\"\014\010\001\020\001\030\001 \002(\0010\001*\0048\002@\001\n\027\030\351\007\"\020\010\001\020\001\030\001 \002(\0010\0018\001@\002*\000 \346\007(\351\007" diff --git a/lib/python3.12/site-packages/google/protobuf/internal/python_message.py b/lib/python3.12/site-packages/google/protobuf/internal/python_message.py new file mode 100644 index 0000000000000000000000000000000000000000..ba79963dd97d4413cfd735e42301bce98fca29bd --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/python_message.py @@ -0,0 +1,1599 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# This code is meant to work on Python 2.4 and above only. +# +# TODO: Helpers for verbose, common checks like seeing if a +# descriptor's cpp_type is CPPTYPE_MESSAGE. + +"""Contains a metaclass and helper functions used to create +protocol message classes from Descriptor objects at runtime. + +Recall that a metaclass is the "type" of a class. +(A class is to a metaclass what an instance is to a class.) + +In this case, we use the GeneratedProtocolMessageType metaclass +to inject all the useful functionality into the classes +output by the protocol compiler at compile-time. + +The upshot of all this is that the real implementation +details for ALL pure-Python protocol buffers are *here in +this file*. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import datetime +from io import BytesIO +import math +import struct +import sys +import warnings +import weakref + +from google.protobuf import descriptor as descriptor_mod +from google.protobuf import message as message_mod +from google.protobuf import text_format +# We use "as" to avoid name collisions with variables. +from google.protobuf.internal import api_implementation +from google.protobuf.internal import containers +from google.protobuf.internal import decoder +from google.protobuf.internal import encoder +from google.protobuf.internal import enum_type_wrapper +from google.protobuf.internal import extension_dict +from google.protobuf.internal import message_listener as message_listener_mod +from google.protobuf.internal import type_checkers +from google.protobuf.internal import well_known_types +from google.protobuf.internal import wire_format + +_FieldDescriptor = descriptor_mod.FieldDescriptor +_AnyFullTypeName = 'google.protobuf.Any' +_StructFullTypeName = 'google.protobuf.Struct' +_ListValueFullTypeName = 'google.protobuf.ListValue' +_ExtensionDict = extension_dict._ExtensionDict + +class GeneratedProtocolMessageType(type): + + """Metaclass for protocol message classes created at runtime from Descriptors. + + We add implementations for all methods described in the Message class. We + also create properties to allow getting/setting all fields in the protocol + message. Finally, we create slots to prevent users from accidentally + "setting" nonexistent fields in the protocol message, which then wouldn't get + serialized / deserialized properly. + + The protocol compiler currently uses this metaclass to create protocol + message classes at runtime. Clients can also manually create their own + classes at runtime, as in this example: + + mydescriptor = Descriptor(.....) + factory = symbol_database.Default() + factory.pool.AddDescriptor(mydescriptor) + MyProtoClass = message_factory.GetMessageClass(mydescriptor) + myproto_instance = MyProtoClass() + myproto.foo_field = 23 + ... + """ + + # Must be consistent with the protocol-compiler code in + # proto2/compiler/internal/generator.*. + _DESCRIPTOR_KEY = 'DESCRIPTOR' + + def __new__(cls, name, bases, dictionary): + """Custom allocation for runtime-generated class types. + + We override __new__ because this is apparently the only place + where we can meaningfully set __slots__ on the class we're creating(?). + (The interplay between metaclasses and slots is not very well-documented). + + Args: + name: Name of the class (ignored, but required by the + metaclass protocol). + bases: Base classes of the class we're constructing. + (Should be message.Message). We ignore this field, but + it's required by the metaclass protocol + dictionary: The class dictionary of the class we're + constructing. dictionary[_DESCRIPTOR_KEY] must contain + a Descriptor object describing this protocol message + type. + + Returns: + Newly-allocated class. + + Raises: + RuntimeError: Generated code only work with python cpp extension. + """ + descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY] + + if isinstance(descriptor, str): + raise RuntimeError('The generated code only work with python cpp ' + 'extension, but it is using pure python runtime.') + + # If a concrete class already exists for this descriptor, don't try to + # create another. Doing so will break any messages that already exist with + # the existing class. + # + # The C++ implementation appears to have its own internal `PyMessageFactory` + # to achieve similar results. + # + # This most commonly happens in `text_format.py` when using descriptors from + # a custom pool; it calls message_factory.GetMessageClass() on a + # descriptor which already has an existing concrete class. + new_class = getattr(descriptor, '_concrete_class', None) + if new_class: + return new_class + + if descriptor.full_name in well_known_types.WKTBASES: + bases += (well_known_types.WKTBASES[descriptor.full_name],) + _AddClassAttributesForNestedExtensions(descriptor, dictionary) + _AddSlots(descriptor, dictionary) + + superclass = super(GeneratedProtocolMessageType, cls) + new_class = superclass.__new__(cls, name, bases, dictionary) + return new_class + + def __init__(cls, name, bases, dictionary): + """Here we perform the majority of our work on the class. + We add enum getters, an __init__ method, implementations + of all Message methods, and properties for all fields + in the protocol type. + + Args: + name: Name of the class (ignored, but required by the + metaclass protocol). + bases: Base classes of the class we're constructing. + (Should be message.Message). We ignore this field, but + it's required by the metaclass protocol + dictionary: The class dictionary of the class we're + constructing. dictionary[_DESCRIPTOR_KEY] must contain + a Descriptor object describing this protocol message + type. + """ + descriptor = dictionary[GeneratedProtocolMessageType._DESCRIPTOR_KEY] + + # If this is an _existing_ class looked up via `_concrete_class` in the + # __new__ method above, then we don't need to re-initialize anything. + existing_class = getattr(descriptor, '_concrete_class', None) + if existing_class: + assert existing_class is cls, ( + 'Duplicate `GeneratedProtocolMessageType` created for descriptor %r' + % (descriptor.full_name)) + return + + cls._message_set_decoders_by_tag = {} + cls._fields_by_tag = {} + if (descriptor.has_options and + descriptor.GetOptions().message_set_wire_format): + cls._message_set_decoders_by_tag[decoder.MESSAGE_SET_ITEM_TAG] = ( + decoder.MessageSetItemDecoder(descriptor), + None, + ) + + # Attach stuff to each FieldDescriptor for quick lookup later on. + for field in descriptor.fields: + _AttachFieldHelpers(cls, field) + + if descriptor.is_extendable and hasattr(descriptor.file, 'pool'): + extensions = descriptor.file.pool.FindAllExtensions(descriptor) + for ext in extensions: + _AttachFieldHelpers(cls, ext) + + descriptor._concrete_class = cls # pylint: disable=protected-access + _AddEnumValues(descriptor, cls) + _AddInitMethod(descriptor, cls) + _AddPropertiesForFields(descriptor, cls) + _AddPropertiesForExtensions(descriptor, cls) + _AddStaticMethods(cls) + _AddMessageMethods(descriptor, cls) + _AddPrivateHelperMethods(descriptor, cls) + + superclass = super(GeneratedProtocolMessageType, cls) + superclass.__init__(name, bases, dictionary) + + +# Stateless helpers for GeneratedProtocolMessageType below. +# Outside clients should not access these directly. +# +# I opted not to make any of these methods on the metaclass, to make it more +# clear that I'm not really using any state there and to keep clients from +# thinking that they have direct access to these construction helpers. + + +def _PropertyName(proto_field_name): + """Returns the name of the public property attribute which + clients can use to get and (in some cases) set the value + of a protocol message field. + + Args: + proto_field_name: The protocol message field name, exactly + as it appears (or would appear) in a .proto file. + """ + # TODO: Escape Python keywords (e.g., yield), and test this support. + # nnorwitz makes my day by writing: + # """ + # FYI. See the keyword module in the stdlib. This could be as simple as: + # + # if keyword.iskeyword(proto_field_name): + # return proto_field_name + "_" + # return proto_field_name + # """ + # Kenton says: The above is a BAD IDEA. People rely on being able to use + # getattr() and setattr() to reflectively manipulate field values. If we + # rename the properties, then every such user has to also make sure to apply + # the same transformation. Note that currently if you name a field "yield", + # you can still access it just fine using getattr/setattr -- it's not even + # that cumbersome to do so. + # TODO: Remove this method entirely if/when everyone agrees with my + # position. + return proto_field_name + + +def _AddSlots(message_descriptor, dictionary): + """Adds a __slots__ entry to dictionary, containing the names of all valid + attributes for this message type. + + Args: + message_descriptor: A Descriptor instance describing this message type. + dictionary: Class dictionary to which we'll add a '__slots__' entry. + """ + dictionary['__slots__'] = ['_cached_byte_size', + '_cached_byte_size_dirty', + '_fields', + '_unknown_fields', + '_is_present_in_parent', + '_listener', + '_listener_for_children', + '__weakref__', + '_oneofs'] + + +def _IsMessageSetExtension(field): + return (field.is_extension and + field.containing_type.has_options and + field.containing_type.GetOptions().message_set_wire_format and + field.type == _FieldDescriptor.TYPE_MESSAGE and + not field.is_required and + not field.is_repeated) + + +def _IsMapField(field): + return (field.type == _FieldDescriptor.TYPE_MESSAGE and + field.message_type._is_map_entry) + + +def _IsMessageMapField(field): + value_type = field.message_type.fields_by_name['value'] + return value_type.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE + +def _AttachFieldHelpers(cls, field_descriptor): + field_descriptor._default_constructor = _DefaultValueConstructorForField( + field_descriptor + ) + + def AddFieldByTag(wiretype, is_packed): + tag_bytes = encoder.TagBytes(field_descriptor.number, wiretype) + cls._fields_by_tag[tag_bytes] = (field_descriptor, is_packed) + + AddFieldByTag( + type_checkers.FIELD_TYPE_TO_WIRE_TYPE[field_descriptor.type], False + ) + + if field_descriptor.is_repeated and wire_format.IsTypePackable( + field_descriptor.type + ): + # To support wire compatibility of adding packed = true, add a decoder for + # packed values regardless of the field's options. + AddFieldByTag(wire_format.WIRETYPE_LENGTH_DELIMITED, True) + + +def _MaybeAddEncoder(cls, field_descriptor): + if hasattr(field_descriptor, '_encoder'): + return + is_repeated = field_descriptor.is_repeated + is_map_entry = _IsMapField(field_descriptor) + is_packed = field_descriptor.is_packed + + if is_map_entry: + field_encoder = encoder.MapEncoder(field_descriptor) + sizer = encoder.MapSizer(field_descriptor, + _IsMessageMapField(field_descriptor)) + elif _IsMessageSetExtension(field_descriptor): + field_encoder = encoder.MessageSetItemEncoder(field_descriptor.number) + sizer = encoder.MessageSetItemSizer(field_descriptor.number) + else: + field_encoder = type_checkers.TYPE_TO_ENCODER[field_descriptor.type]( + field_descriptor.number, is_repeated, is_packed) + sizer = type_checkers.TYPE_TO_SIZER[field_descriptor.type]( + field_descriptor.number, is_repeated, is_packed) + + field_descriptor._sizer = sizer + field_descriptor._encoder = field_encoder + + +def _MaybeAddDecoder(cls, field_descriptor): + if hasattr(field_descriptor, '_decoders'): + return + + is_repeated = field_descriptor.is_repeated + is_map_entry = _IsMapField(field_descriptor) + helper_decoders = {} + + def AddDecoder(is_packed): + decode_type = field_descriptor.type + if (decode_type == _FieldDescriptor.TYPE_ENUM and + not field_descriptor.enum_type.is_closed): + decode_type = _FieldDescriptor.TYPE_INT32 + + oneof_descriptor = None + if field_descriptor.containing_oneof is not None: + oneof_descriptor = field_descriptor + + if is_map_entry: + is_message_map = _IsMessageMapField(field_descriptor) + + field_decoder = decoder.MapDecoder( + field_descriptor, _GetInitializeDefaultForMap(field_descriptor), + is_message_map) + elif decode_type == _FieldDescriptor.TYPE_STRING: + field_decoder = decoder.StringDecoder( + field_descriptor.number, is_repeated, is_packed, + field_descriptor, field_descriptor._default_constructor, + not field_descriptor.has_presence) + elif field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + field_decoder = type_checkers.TYPE_TO_DECODER[decode_type]( + field_descriptor.number, is_repeated, is_packed, + field_descriptor, field_descriptor._default_constructor) + else: + field_decoder = type_checkers.TYPE_TO_DECODER[decode_type]( + field_descriptor.number, is_repeated, is_packed, + # pylint: disable=protected-access + field_descriptor, field_descriptor._default_constructor, + not field_descriptor.has_presence) + + helper_decoders[is_packed] = field_decoder + + AddDecoder(False) + + if is_repeated and wire_format.IsTypePackable(field_descriptor.type): + # To support wire compatibility of adding packed = true, add a decoder for + # packed values regardless of the field's options. + AddDecoder(True) + + field_descriptor._decoders = helper_decoders + + +def _AddClassAttributesForNestedExtensions(descriptor, dictionary): + extensions = descriptor.extensions_by_name + for extension_name, extension_field in extensions.items(): + assert extension_name not in dictionary + dictionary[extension_name] = extension_field + + +def _AddEnumValues(descriptor, cls): + """Sets class-level attributes for all enum fields defined in this message. + + Also exporting a class-level object that can name enum values. + + Args: + descriptor: Descriptor object for this message type. + cls: Class we're constructing for this message type. + """ + for enum_type in descriptor.enum_types: + setattr(cls, enum_type.name, enum_type_wrapper.EnumTypeWrapper(enum_type)) + for enum_value in enum_type.values: + setattr(cls, enum_value.name, enum_value.number) + + +def _GetInitializeDefaultForMap(field): + if not field.is_repeated: + raise ValueError('map_entry set on non-repeated field %s' % ( + field.name)) + fields_by_name = field.message_type.fields_by_name + key_checker = type_checkers.GetTypeChecker(fields_by_name['key']) + + value_field = fields_by_name['value'] + if _IsMessageMapField(field): + def MakeMessageMapDefault(message): + return containers.MessageMap( + message._listener_for_children, value_field.message_type, key_checker, + field.message_type) + return MakeMessageMapDefault + else: + value_checker = type_checkers.GetTypeChecker(value_field) + def MakePrimitiveMapDefault(message): + return containers.ScalarMap( + message._listener_for_children, key_checker, value_checker, + field.message_type) + return MakePrimitiveMapDefault + +def _DefaultValueConstructorForField(field): + """Returns a function which returns a default value for a field. + + Args: + field: FieldDescriptor object for this field. + + The returned function has one argument: + message: Message instance containing this field, or a weakref proxy + of same. + + That function in turn returns a default value for this field. The default + value may refer back to |message| via a weak reference. + """ + + if _IsMapField(field): + return _GetInitializeDefaultForMap(field) + + if field.is_repeated: + if field.has_default_value and field.default_value != []: + raise ValueError('Repeated field default value not empty list: %s' % ( + field.default_value)) + if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + # We can't look at _concrete_class yet since it might not have + # been set. (Depends on order in which we initialize the classes). + message_type = field.message_type + def MakeRepeatedMessageDefault(message): + return containers.RepeatedCompositeFieldContainer( + message._listener_for_children, field.message_type) + return MakeRepeatedMessageDefault + else: + type_checker = type_checkers.GetTypeChecker(field) + def MakeRepeatedScalarDefault(message): + return containers.RepeatedScalarFieldContainer( + message._listener_for_children, type_checker) + return MakeRepeatedScalarDefault + + if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + message_type = field.message_type + def MakeSubMessageDefault(message): + # _concrete_class may not yet be initialized. + if not hasattr(message_type, '_concrete_class'): + from google.protobuf import message_factory + message_factory.GetMessageClass(message_type) + result = message_type._concrete_class() + result._SetListener( + _OneofListener(message, field) + if field.containing_oneof is not None + else message._listener_for_children) + return result + return MakeSubMessageDefault + + def MakeScalarDefault(message): + # TODO: This may be broken since there may not be + # default_value. Combine with has_default_value somehow. + return field.default_value + return MakeScalarDefault + + +def _ReraiseTypeErrorWithFieldName(message_name, field_name): + """Re-raise the currently-handled TypeError with the field name added.""" + exc = sys.exc_info()[1] + if len(exc.args) == 1 and type(exc) is TypeError: + # simple TypeError; add field name to exception message + exc = TypeError('%s for field %s.%s' % (str(exc), message_name, field_name)) + + # re-raise possibly-amended exception with original traceback: + raise exc.with_traceback(sys.exc_info()[2]) + + +def _AddInitMethod(message_descriptor, cls): + """Adds an __init__ method to cls.""" + + def _GetIntegerEnumValue(enum_type, value): + """Convert a string or integer enum value to an integer. + + If the value is a string, it is converted to the enum value in + enum_type with the same name. If the value is not a string, it's + returned as-is. (No conversion or bounds-checking is done.) + """ + if isinstance(value, str): + try: + return enum_type.values_by_name[value].number + except KeyError: + raise ValueError('Enum type %s: unknown label "%s"' % ( + enum_type.full_name, value)) + return value + + def init(self, **kwargs): + + def init_wkt_or_merge(field, msg, value): + if isinstance(value, message_mod.Message): + msg.MergeFrom(value) + elif ( + isinstance(value, dict) + and field.message_type.full_name == _StructFullTypeName + ): + msg.Clear() + if len(value) == 1 and 'fields' in value: + try: + msg.update(value) + except: + msg.Clear() + msg.__init__(**value) + else: + msg.update(value) + elif hasattr(msg, '_internal_assign'): + msg._internal_assign(value) + else: + raise TypeError( + 'Message field {0}.{1} must be initialized with a ' + 'dict or instance of same class, got {2}.'.format( + message_descriptor.name, + field.name, + type(value).__name__, + ) + ) + + self._cached_byte_size = 0 + self._cached_byte_size_dirty = len(kwargs) > 0 + self._fields = {} + # Contains a mapping from oneof field descriptors to the descriptor + # of the currently set field in that oneof field. + self._oneofs = {} + + # _unknown_fields is () when empty for efficiency, and will be turned into + # a list if fields are added. + self._unknown_fields = () + self._is_present_in_parent = False + self._listener = message_listener_mod.NullMessageListener() + self._listener_for_children = _Listener(self) + for field_name, field_value in kwargs.items(): + field = _GetFieldByName(message_descriptor, field_name) + if field is None: + raise TypeError('%s() got an unexpected keyword argument "%s"' % + (message_descriptor.name, field_name)) + if field_value is None: + # field=None is the same as no field at all. + continue + if field.is_repeated: + field_copy = field._default_constructor(self) + if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: # Composite + if _IsMapField(field): + if _IsMessageMapField(field): + for key in field_value: + item_value = field_value[key] + if isinstance(item_value, dict): + field_copy[key].__init__(**item_value) + else: + field_copy[key].MergeFrom(item_value) + else: + field_copy.update(field_value) + else: + for val in field_value: + if isinstance(val, dict) and ( + field.message_type.full_name != _StructFullTypeName + ): + field_copy.add(**val) + else: + new_msg = field_copy.add() + init_wkt_or_merge(field, new_msg, val) + else: # Scalar + if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM: + field_value = [_GetIntegerEnumValue(field.enum_type, val) + for val in field_value] + field_copy.extend(field_value) + self._fields[field] = field_copy + elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + field_copy = field._default_constructor(self) + if isinstance(field_value, dict) and ( + field.message_type.full_name != _StructFullTypeName + ): + new_val = field.message_type._concrete_class(**field_value) + field_copy.MergeFrom(new_val) + else: + try: + init_wkt_or_merge(field, field_copy, field_value) + except TypeError: + _ReraiseTypeErrorWithFieldName(message_descriptor.name, field_name) + self._fields[field] = field_copy + else: + if field.cpp_type == _FieldDescriptor.CPPTYPE_ENUM: + field_value = _GetIntegerEnumValue(field.enum_type, field_value) + try: + setattr(self, field_name, field_value) + except TypeError: + _ReraiseTypeErrorWithFieldName(message_descriptor.name, field_name) + + init.__module__ = None + init.__doc__ = None + cls.__init__ = init + + +def _GetFieldByName(message_descriptor, field_name): + """Returns a field descriptor by field name. + + Args: + message_descriptor: A Descriptor describing all fields in message. + field_name: The name of the field to retrieve. + Returns: + The field descriptor associated with the field name. + """ + try: + return message_descriptor.fields_by_name[field_name] + except KeyError: + raise ValueError('Protocol message %s has no "%s" field.' % + (message_descriptor.name, field_name)) + + +def _AddPropertiesForFields(descriptor, cls): + """Adds properties for all fields in this protocol message type.""" + for field in descriptor.fields: + _AddPropertiesForField(field, cls) + + if descriptor.is_extendable: + # _ExtensionDict is just an adaptor with no state so we allocate a new one + # every time it is accessed. + cls.Extensions = property(lambda self: _ExtensionDict(self)) + + +def _AddPropertiesForField(field, cls): + """Adds a public property for a protocol message field. + Clients can use this property to get and (in the case + of non-repeated scalar fields) directly set the value + of a protocol message field. + + Args: + field: A FieldDescriptor for this field. + cls: The class we're constructing. + """ + # Catch it if we add other types that we should + # handle specially here. + assert _FieldDescriptor.MAX_CPPTYPE == 10 + + constant_name = field.name.upper() + '_FIELD_NUMBER' + setattr(cls, constant_name, field.number) + + if field.is_repeated: + _AddPropertiesForRepeatedField(field, cls) + elif field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + _AddPropertiesForNonRepeatedCompositeField(field, cls) + else: + _AddPropertiesForNonRepeatedScalarField(field, cls) + + +class _FieldProperty(property): + __slots__ = ('DESCRIPTOR',) + + def __init__(self, descriptor, getter, setter, doc): + property.__init__(self, getter, setter, doc=doc) + self.DESCRIPTOR = descriptor + + +def _AddPropertiesForRepeatedField(field, cls): + """Adds a public property for a "repeated" protocol message field. Clients + can use this property to get the value of the field, which will be either a + RepeatedScalarFieldContainer or RepeatedCompositeFieldContainer (see + below). + + Note that when clients add values to these containers, we perform + type-checking in the case of repeated scalar fields, and we also set any + necessary "has" bits as a side-effect. + + Args: + field: A FieldDescriptor for this field. + cls: The class we're constructing. + """ + proto_field_name = field.name + property_name = _PropertyName(proto_field_name) + + def getter(self): + field_value = self._fields.get(field) + if field_value is None: + # Construct a new object to represent this field. + field_value = field._default_constructor(self) + + # Atomically check if another thread has preempted us and, if not, swap + # in the new object we just created. If someone has preempted us, we + # take that object and discard ours. + # WARNING: We are relying on setdefault() being atomic. This is true + # in CPython but we haven't investigated others. This warning appears + # in several other locations in this file. + field_value = self._fields.setdefault(field, field_value) + return field_value + getter.__module__ = None + getter.__doc__ = 'Getter for %s.' % proto_field_name + + # We define a setter just so we can throw an exception with a more + # helpful error message. + def setter(self, new_value): + raise AttributeError('Assignment not allowed to repeated field ' + '"%s" in protocol message object.' % proto_field_name) + + doc = 'Magic attribute generated for "%s" proto field.' % proto_field_name + setattr(cls, property_name, _FieldProperty(field, getter, setter, doc=doc)) + + +def _AddPropertiesForNonRepeatedScalarField(field, cls): + """Adds a public property for a nonrepeated, scalar protocol message field. + Clients can use this property to get and directly set the value of the field. + Note that when the client sets the value of a field by using this property, + all necessary "has" bits are set as a side-effect, and we also perform + type-checking. + + Args: + field: A FieldDescriptor for this field. + cls: The class we're constructing. + """ + proto_field_name = field.name + property_name = _PropertyName(proto_field_name) + type_checker = type_checkers.GetTypeChecker(field) + default_value = field.default_value + + def getter(self): + # TODO: This may be broken since there may not be + # default_value. Combine with has_default_value somehow. + return self._fields.get(field, default_value) + getter.__module__ = None + getter.__doc__ = 'Getter for %s.' % proto_field_name + + def field_setter(self, new_value): + # pylint: disable=protected-access + # Testing the value for truthiness captures all of the implicit presence + # defaults (0, 0.0, enum 0, and False), except for -0.0. + try: + new_value = type_checker.CheckValue(new_value) + except TypeError as e: + raise TypeError( + 'Cannot set %s to %.1024r: %s' % (field.full_name, new_value, e)) + if not field.has_presence and decoder.IsDefaultScalarValue(new_value): + self._fields.pop(field, None) + else: + self._fields[field] = new_value + # Check _cached_byte_size_dirty inline to improve performance, since scalar + # setters are called frequently. + if not self._cached_byte_size_dirty: + self._Modified() + + if field.containing_oneof: + def setter(self, new_value): + field_setter(self, new_value) + self._UpdateOneofState(field) + else: + setter = field_setter + + setter.__module__ = None + setter.__doc__ = 'Setter for %s.' % proto_field_name + + # Add a property to encapsulate the getter/setter. + doc = 'Magic attribute generated for "%s" proto field.' % proto_field_name + setattr(cls, property_name, _FieldProperty(field, getter, setter, doc=doc)) + + +def _AddPropertiesForNonRepeatedCompositeField(field, cls): + """Adds a public property for a nonrepeated, composite protocol message field. + A composite field is a "group" or "message" field. + + Clients can use this property to get the value of the field, but cannot + assign to the property directly. + + Args: + field: A FieldDescriptor for this field. + cls: The class we're constructing. + """ + # TODO: Remove duplication with similar method + # for non-repeated scalars. + proto_field_name = field.name + property_name = _PropertyName(proto_field_name) + + def getter(self): + field_value = self._fields.get(field) + if field_value is None: + # Construct a new object to represent this field. + field_value = field._default_constructor(self) + + # Atomically check if another thread has preempted us and, if not, swap + # in the new object we just created. If someone has preempted us, we + # take that object and discard ours. + # WARNING: We are relying on setdefault() being atomic. This is true + # in CPython but we haven't investigated others. This warning appears + # in several other locations in this file. + field_value = self._fields.setdefault(field, field_value) + return field_value + getter.__module__ = None + getter.__doc__ = 'Getter for %s.' % proto_field_name + + # We define a setter just so we can throw an exception with a more + # helpful error message. + def setter(self, new_value): + if field.message_type.full_name == 'google.protobuf.Timestamp': + getter(self) + self._fields[field].FromDatetime(new_value) + elif field.message_type.full_name == 'google.protobuf.Duration': + getter(self) + self._fields[field].FromTimedelta(new_value) + elif field.message_type.full_name == _StructFullTypeName: + getter(self) + self._fields[field].Clear() + self._fields[field].update(new_value) + elif field.message_type.full_name == _ListValueFullTypeName: + getter(self) + self._fields[field].Clear() + self._fields[field].extend(new_value) + else: + raise AttributeError( + 'Assignment not allowed to composite field ' + '"%s" in protocol message object.' % proto_field_name + ) + + # Add a property to encapsulate the getter. + doc = 'Magic attribute generated for "%s" proto field.' % proto_field_name + setattr(cls, property_name, _FieldProperty(field, getter, setter, doc=doc)) + + +def _AddPropertiesForExtensions(descriptor, cls): + """Adds properties for all fields in this protocol message type.""" + extensions = descriptor.extensions_by_name + for extension_name, extension_field in extensions.items(): + constant_name = extension_name.upper() + '_FIELD_NUMBER' + setattr(cls, constant_name, extension_field.number) + + # TODO: Migrate all users of these attributes to functions like + # pool.FindExtensionByNumber(descriptor). + if descriptor.file is not None: + # TODO: Use cls.MESSAGE_FACTORY.pool when available. + pool = descriptor.file.pool + +def _AddStaticMethods(cls): + + def RegisterExtension(_): + """no-op to keep generated code <=4.23 working with new runtimes.""" + # This was originally removed in 5.26 (cl/595989309). + pass + + cls.RegisterExtension = staticmethod(RegisterExtension) + def FromString(s): + message = cls() + message.MergeFromString(s) + return message + cls.FromString = staticmethod(FromString) + + +def _IsPresent(item): + """Given a (FieldDescriptor, value) tuple from _fields, return true if the + value should be included in the list returned by ListFields().""" + + if item[0].is_repeated: + return bool(item[1]) + elif item[0].cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + return item[1]._is_present_in_parent + else: + return True + + +def _AddListFieldsMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + + def ListFields(self): + all_fields = [item for item in self._fields.items() if _IsPresent(item)] + all_fields.sort(key = lambda item: item[0].number) + return all_fields + + cls.ListFields = ListFields + + +def _AddHasFieldMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + + hassable_fields = {} + for field in message_descriptor.fields: + if field.is_repeated: + continue + # For proto3, only submessages and fields inside a oneof have presence. + if not field.has_presence: + continue + hassable_fields[field.name] = field + + # Has methods are supported for oneof descriptors. + for oneof in message_descriptor.oneofs: + hassable_fields[oneof.name] = oneof + + def HasField(self, field_name): + try: + field = hassable_fields[field_name] + except KeyError as exc: + raise ValueError('Protocol message %s has no non-repeated field "%s" ' + 'nor has presence is not available for this field.' % ( + message_descriptor.full_name, field_name)) from exc + + if isinstance(field, descriptor_mod.OneofDescriptor): + try: + return HasField(self, self._oneofs[field].name) + except KeyError: + return False + else: + if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + value = self._fields.get(field) + return value is not None and value._is_present_in_parent + else: + return field in self._fields + + cls.HasField = HasField + + +def _AddClearFieldMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + def ClearField(self, field_name): + try: + field = message_descriptor.fields_by_name[field_name] + except KeyError: + try: + field = message_descriptor.oneofs_by_name[field_name] + if field in self._oneofs: + field = self._oneofs[field] + else: + return + except KeyError: + raise ValueError('Protocol message %s has no "%s" field.' % + (message_descriptor.name, field_name)) + + if field in self._fields: + # To match the C++ implementation, we need to invalidate iterators + # for map fields when ClearField() happens. + if hasattr(self._fields[field], 'InvalidateIterators'): + self._fields[field].InvalidateIterators() + + # Note: If the field is a sub-message, its listener will still point + # at us. That's fine, because the worst than can happen is that it + # will call _Modified() and invalidate our byte size. Big deal. + del self._fields[field] + + if self._oneofs.get(field.containing_oneof, None) is field: + del self._oneofs[field.containing_oneof] + + # Always call _Modified() -- even if nothing was changed, this is + # a mutating method, and thus calling it should cause the field to become + # present in the parent message. + self._Modified() + + cls.ClearField = ClearField + + +def _AddClearExtensionMethod(cls): + """Helper for _AddMessageMethods().""" + def ClearExtension(self, field_descriptor): + extension_dict._VerifyExtensionHandle(self, field_descriptor) + + # Similar to ClearField(), above. + if field_descriptor in self._fields: + del self._fields[field_descriptor] + self._Modified() + cls.ClearExtension = ClearExtension + + +def _AddHasExtensionMethod(cls): + """Helper for _AddMessageMethods().""" + def HasExtension(self, field_descriptor): + extension_dict._VerifyExtensionHandle(self, field_descriptor) + if field_descriptor.is_repeated: + raise KeyError('"%s" is repeated.' % field_descriptor.full_name) + + if field_descriptor.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + value = self._fields.get(field_descriptor) + return value is not None and value._is_present_in_parent + else: + return field_descriptor in self._fields + cls.HasExtension = HasExtension + +def _InternalUnpackAny(msg): + """Unpacks Any message and returns the unpacked message. + + This internal method is different from public Any Unpack method which takes + the target message as argument. _InternalUnpackAny method does not have + target message type and need to find the message type in descriptor pool. + + Args: + msg: An Any message to be unpacked. + + Returns: + The unpacked message. + """ + # TODO: Don't use the factory of generated messages. + # To make Any work with custom factories, use the message factory of the + # parent message. + # pylint: disable=g-import-not-at-top + from google.protobuf import symbol_database + factory = symbol_database.Default() + + type_url = msg.type_url + + if not type_url: + return None + + # TODO: For now we just strip the hostname. Better logic will be + # required. + type_name = type_url.split('/')[-1] + descriptor = factory.pool.FindMessageTypeByName(type_name) + + if descriptor is None: + return None + + # Unable to import message_factory at top because of circular import. + # pylint: disable=g-import-not-at-top + from google.protobuf import message_factory + message_class = message_factory.GetMessageClass(descriptor) + message = message_class() + + message.ParseFromString(msg.value) + return message + + +def _AddEqualsMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + def __eq__(self, other): + if self.DESCRIPTOR.full_name == _ListValueFullTypeName and isinstance( + other, list + ): + return self._internal_compare(other) + if self.DESCRIPTOR.full_name == _StructFullTypeName and isinstance( + other, dict + ): + return self._internal_compare(other) + + if (not isinstance(other, message_mod.Message) or + other.DESCRIPTOR != self.DESCRIPTOR): + return NotImplemented + + if self is other: + return True + + if self.DESCRIPTOR.full_name == _AnyFullTypeName: + any_a = _InternalUnpackAny(self) + any_b = _InternalUnpackAny(other) + if any_a and any_b: + return any_a == any_b + + if not self.ListFields() == other.ListFields(): + return False + + # TODO: Fix UnknownFieldSet to consider MessageSet extensions, + # then use it for the comparison. + unknown_fields = list(self._unknown_fields) + unknown_fields.sort() + other_unknown_fields = list(other._unknown_fields) + other_unknown_fields.sort() + return unknown_fields == other_unknown_fields + + cls.__eq__ = __eq__ + + +def _AddStrMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + def __str__(self): + return text_format.MessageToString(self) + cls.__str__ = __str__ + + +def _AddReprMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + def __repr__(self): + return text_format.MessageToString(self) + cls.__repr__ = __repr__ + + +def _AddUnicodeMethod(unused_message_descriptor, cls): + """Helper for _AddMessageMethods().""" + + def __unicode__(self): + return text_format.MessageToString(self, as_utf8=True).decode('utf-8') + cls.__unicode__ = __unicode__ + + +def _AddContainsMethod(message_descriptor, cls): + + if message_descriptor.full_name == 'google.protobuf.Struct': + def __contains__(self, key): + return key in self.fields + elif message_descriptor.full_name == 'google.protobuf.ListValue': + def __contains__(self, value): + return value in self.items() + else: + def __contains__(self, field): + return self.HasField(field) + + cls.__contains__ = __contains__ + + +def _BytesForNonRepeatedElement(value, field_number, field_type): + """Returns the number of bytes needed to serialize a non-repeated element. + The returned byte count includes space for tag information and any + other additional space associated with serializing value. + + Args: + value: Value we're serializing. + field_number: Field number of this value. (Since the field number + is stored as part of a varint-encoded tag, this has an impact + on the total bytes required to serialize the value). + field_type: The type of the field. One of the TYPE_* constants + within FieldDescriptor. + """ + try: + fn = type_checkers.TYPE_TO_BYTE_SIZE_FN[field_type] + return fn(field_number, value) + except KeyError: + raise message_mod.EncodeError('Unrecognized field type: %d' % field_type) + + +def _AddByteSizeMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + + def ByteSize(self): + if not self._cached_byte_size_dirty: + return self._cached_byte_size + + size = 0 + descriptor = self.DESCRIPTOR + if descriptor._is_map_entry: + # Fields of map entry should always be serialized. + key_field = descriptor.fields_by_name['key'] + _MaybeAddEncoder(cls, key_field) + size = key_field._sizer(self.key) + value_field = descriptor.fields_by_name['value'] + _MaybeAddEncoder(cls, value_field) + size += value_field._sizer(self.value) + else: + for field_descriptor, field_value in self.ListFields(): + _MaybeAddEncoder(cls, field_descriptor) + size += field_descriptor._sizer(field_value) + for tag_bytes, value_bytes in self._unknown_fields: + size += len(tag_bytes) + len(value_bytes) + + self._cached_byte_size = size + self._cached_byte_size_dirty = False + self._listener_for_children.dirty = False + return size + + cls.ByteSize = ByteSize + + +def _AddSerializeToStringMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + + def SerializeToString(self, **kwargs): + # Check if the message has all of its required fields set. + if not self.IsInitialized(): + raise message_mod.EncodeError( + 'Message %s is missing required fields: %s' % ( + self.DESCRIPTOR.full_name, ','.join(self.FindInitializationErrors()))) + return self.SerializePartialToString(**kwargs) + cls.SerializeToString = SerializeToString + + +def _AddSerializePartialToStringMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + + def SerializePartialToString(self, **kwargs): + out = BytesIO() + self._InternalSerialize(out.write, **kwargs) + return out.getvalue() + cls.SerializePartialToString = SerializePartialToString + + def InternalSerialize(self, write_bytes, deterministic=None): + if deterministic is None: + deterministic = ( + api_implementation.IsPythonDefaultSerializationDeterministic()) + else: + deterministic = bool(deterministic) + + descriptor = self.DESCRIPTOR + if descriptor._is_map_entry: + # Fields of map entry should always be serialized. + key_field = descriptor.fields_by_name['key'] + _MaybeAddEncoder(cls, key_field) + key_field._encoder(write_bytes, self.key, deterministic) + value_field = descriptor.fields_by_name['value'] + _MaybeAddEncoder(cls, value_field) + value_field._encoder(write_bytes, self.value, deterministic) + else: + for field_descriptor, field_value in self.ListFields(): + _MaybeAddEncoder(cls, field_descriptor) + field_descriptor._encoder(write_bytes, field_value, deterministic) + for tag_bytes, value_bytes in self._unknown_fields: + write_bytes(tag_bytes) + write_bytes(value_bytes) + cls._InternalSerialize = InternalSerialize + + +def _AddMergeFromStringMethod(message_descriptor, cls): + """Helper for _AddMessageMethods().""" + def MergeFromString(self, serialized): + serialized = memoryview(serialized) + length = len(serialized) + try: + if self._InternalParse(serialized, 0, length) != length: + # The only reason _InternalParse would return early is if it + # encountered an end-group tag. + raise message_mod.DecodeError('Unexpected end-group tag.') + except (IndexError, TypeError): + # Now ord(buf[p:p+1]) == ord('') gets TypeError. + raise message_mod.DecodeError('Truncated message.') + except struct.error as e: + raise message_mod.DecodeError(e) + return length # Return this for legacy reasons. + cls.MergeFromString = MergeFromString + + fields_by_tag = cls._fields_by_tag + message_set_decoders_by_tag = cls._message_set_decoders_by_tag + + def InternalParse(self, buffer, pos, end, current_depth=0): + """Create a message from serialized bytes. + + Args: + self: Message, instance of the proto message object. + buffer: memoryview of the serialized data. + pos: int, position to start in the serialized data. + end: int, end position of the serialized data. + + Returns: + Message object. + """ + # Guard against internal misuse, since this function is called internally + # quite extensively, and its easy to accidentally pass bytes. + assert isinstance(buffer, memoryview) + self._Modified() + field_dict = self._fields + while pos != end: + (tag_bytes, new_pos) = decoder.ReadTag(buffer, pos) + field_decoder, field_des = message_set_decoders_by_tag.get( + tag_bytes, (None, None) + ) + if field_decoder: + pos = field_decoder(buffer, new_pos, end, self, field_dict) + continue + field_des, is_packed = fields_by_tag.get(tag_bytes, (None, None)) + if field_des is None: + if not self._unknown_fields: # pylint: disable=protected-access + self._unknown_fields = [] # pylint: disable=protected-access + field_number, wire_type = decoder.DecodeTag(tag_bytes) + if field_number == 0: + raise message_mod.DecodeError('Field number 0 is illegal.') + (data, new_pos) = decoder._DecodeUnknownField( + buffer, new_pos, end, field_number, wire_type + ) # pylint: disable=protected-access + if new_pos == -1: + return pos + self._unknown_fields.append( + (tag_bytes, buffer[pos + len(tag_bytes) : new_pos].tobytes()) + ) + pos = new_pos + else: + _MaybeAddDecoder(cls, field_des) + field_decoder = field_des._decoders[is_packed] + pos = field_decoder( + buffer, new_pos, end, self, field_dict, current_depth + ) + if field_des.containing_oneof: + self._UpdateOneofState(field_des) + return pos + + cls._InternalParse = InternalParse + + +def _AddIsInitializedMethod(message_descriptor, cls): + """Adds the IsInitialized and FindInitializationError methods to the + protocol message class.""" + + required_fields = [field for field in message_descriptor.fields + if field.is_required] + + def IsInitialized(self, errors=None): + """Checks if all required fields of a message are set. + + Args: + errors: A list which, if provided, will be populated with the field + paths of all missing required fields. + + Returns: + True iff the specified message has all required fields set. + """ + + # Performance is critical so we avoid HasField() and ListFields(). + + for field in required_fields: + if (field not in self._fields or + (field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE and + not self._fields[field]._is_present_in_parent)): + if errors is not None: + errors.extend(self.FindInitializationErrors()) + return False + + for field, value in list(self._fields.items()): # dict can change size! + if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + if field.is_repeated: + if (field.message_type._is_map_entry): + continue + for element in value: + if not element.IsInitialized(): + if errors is not None: + errors.extend(self.FindInitializationErrors()) + return False + elif value._is_present_in_parent and not value.IsInitialized(): + if errors is not None: + errors.extend(self.FindInitializationErrors()) + return False + + return True + + cls.IsInitialized = IsInitialized + + def FindInitializationErrors(self): + """Finds required fields which are not initialized. + + Returns: + A list of strings. Each string is a path to an uninitialized field from + the top-level message, e.g. "foo.bar[5].baz". + """ + + errors = [] # simplify things + + for field in required_fields: + if not self.HasField(field.name): + errors.append(field.name) + + for field, value in self.ListFields(): + if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + if field.is_extension: + name = '(%s)' % field.full_name + else: + name = field.name + + if _IsMapField(field): + if _IsMessageMapField(field): + for key in value: + element = value[key] + prefix = '%s[%s].' % (name, key) + sub_errors = element.FindInitializationErrors() + errors += [prefix + error for error in sub_errors] + else: + # ScalarMaps can't have any initialization errors. + pass + elif field.is_repeated: + for i in range(len(value)): + element = value[i] + prefix = '%s[%d].' % (name, i) + sub_errors = element.FindInitializationErrors() + errors += [prefix + error for error in sub_errors] + else: + prefix = name + '.' + sub_errors = value.FindInitializationErrors() + errors += [prefix + error for error in sub_errors] + + return errors + + cls.FindInitializationErrors = FindInitializationErrors + + +def _FullyQualifiedClassName(klass): + module = klass.__module__ + name = getattr(klass, '__qualname__', klass.__name__) + if module in (None, 'builtins', '__builtin__'): + return name + return module + '.' + name + + +def _AddMergeFromMethod(cls): + CPPTYPE_MESSAGE = _FieldDescriptor.CPPTYPE_MESSAGE + + def MergeFrom(self, msg): + if not isinstance(msg, cls): + raise TypeError( + 'Parameter to MergeFrom() must be instance of same class: ' + 'expected %s got %s.' % (_FullyQualifiedClassName(cls), + _FullyQualifiedClassName(msg.__class__))) + + assert msg is not self + self._Modified() + + fields = self._fields + + for field, value in msg._fields.items(): + if field.is_repeated: + field_value = fields.get(field) + if field_value is None: + # Construct a new object to represent this field. + field_value = field._default_constructor(self) + fields[field] = field_value + field_value.MergeFrom(value) + elif field.cpp_type == CPPTYPE_MESSAGE: + if value._is_present_in_parent: + field_value = fields.get(field) + if field_value is None: + # Construct a new object to represent this field. + field_value = field._default_constructor(self) + fields[field] = field_value + field_value.MergeFrom(value) + else: + self._fields[field] = value + if field.containing_oneof: + self._UpdateOneofState(field) + + if msg._unknown_fields: + if not self._unknown_fields: + self._unknown_fields = [] + self._unknown_fields.extend(msg._unknown_fields) + + cls.MergeFrom = MergeFrom + + +def _AddWhichOneofMethod(message_descriptor, cls): + def WhichOneof(self, oneof_name): + """Returns the name of the currently set field inside a oneof, or None.""" + try: + field = message_descriptor.oneofs_by_name[oneof_name] + except KeyError: + raise ValueError( + 'Protocol message has no oneof "%s" field.' % oneof_name) + + nested_field = self._oneofs.get(field, None) + if nested_field is not None and self.HasField(nested_field.name): + return nested_field.name + else: + return None + + cls.WhichOneof = WhichOneof + + +def _Clear(self): + # Clear fields. + self._fields = {} + self._unknown_fields = () + + self._oneofs = {} + self._Modified() + + +def _UnknownFields(self): + raise NotImplementedError('Please use the add-on feaure ' + 'unknown_fields.UnknownFieldSet(message) in ' + 'unknown_fields.py instead.') + + +def _DiscardUnknownFields(self): + self._unknown_fields = [] + for field, value in self.ListFields(): + if field.cpp_type == _FieldDescriptor.CPPTYPE_MESSAGE: + if _IsMapField(field): + if _IsMessageMapField(field): + for key in value: + value[key].DiscardUnknownFields() + elif field.is_repeated: + for sub_message in value: + sub_message.DiscardUnknownFields() + else: + value.DiscardUnknownFields() + + +def _SetListener(self, listener): + if listener is None: + self._listener = message_listener_mod.NullMessageListener() + else: + self._listener = listener + + +def _AddMessageMethods(message_descriptor, cls): + """Adds implementations of all Message methods to cls.""" + _AddListFieldsMethod(message_descriptor, cls) + _AddHasFieldMethod(message_descriptor, cls) + _AddClearFieldMethod(message_descriptor, cls) + if message_descriptor.is_extendable: + _AddClearExtensionMethod(cls) + _AddHasExtensionMethod(cls) + _AddEqualsMethod(message_descriptor, cls) + _AddStrMethod(message_descriptor, cls) + _AddReprMethod(message_descriptor, cls) + _AddUnicodeMethod(message_descriptor, cls) + _AddContainsMethod(message_descriptor, cls) + _AddByteSizeMethod(message_descriptor, cls) + _AddSerializeToStringMethod(message_descriptor, cls) + _AddSerializePartialToStringMethod(message_descriptor, cls) + _AddMergeFromStringMethod(message_descriptor, cls) + _AddIsInitializedMethod(message_descriptor, cls) + _AddMergeFromMethod(cls) + _AddWhichOneofMethod(message_descriptor, cls) + # Adds methods which do not depend on cls. + cls.Clear = _Clear + cls.DiscardUnknownFields = _DiscardUnknownFields + cls._SetListener = _SetListener + + +def _AddPrivateHelperMethods(message_descriptor, cls): + """Adds implementation of private helper methods to cls.""" + + def Modified(self): + """Sets the _cached_byte_size_dirty bit to true, + and propagates this to our listener iff this was a state change. + """ + + # Note: Some callers check _cached_byte_size_dirty before calling + # _Modified() as an extra optimization. So, if this method is ever + # changed such that it does stuff even when _cached_byte_size_dirty is + # already true, the callers need to be updated. + if not self._cached_byte_size_dirty: + self._cached_byte_size_dirty = True + self._listener_for_children.dirty = True + self._is_present_in_parent = True + self._listener.Modified() + + def _UpdateOneofState(self, field): + """Sets field as the active field in its containing oneof. + + Will also delete currently active field in the oneof, if it is different + from the argument. Does not mark the message as modified. + """ + other_field = self._oneofs.setdefault(field.containing_oneof, field) + if other_field is not field: + del self._fields[other_field] + self._oneofs[field.containing_oneof] = field + + cls._Modified = Modified + cls.SetInParent = Modified + cls._UpdateOneofState = _UpdateOneofState + + +class _Listener(object): + + """MessageListener implementation that a parent message registers with its + child message. + + In order to support semantics like: + + foo.bar.baz.moo = 23 + assert foo.HasField('bar') + + ...child objects must have back references to their parents. + This helper class is at the heart of this support. + """ + + def __init__(self, parent_message): + """Args: + parent_message: The message whose _Modified() method we should call when + we receive Modified() messages. + """ + # This listener establishes a back reference from a child (contained) object + # to its parent (containing) object. We make this a weak reference to avoid + # creating cyclic garbage when the client finishes with the 'parent' object + # in the tree. + if isinstance(parent_message, weakref.ProxyType): + self._parent_message_weakref = parent_message + else: + self._parent_message_weakref = weakref.proxy(parent_message) + + # As an optimization, we also indicate directly on the listener whether + # or not the parent message is dirty. This way we can avoid traversing + # up the tree in the common case. + self.dirty = False + + def Modified(self): + if self.dirty: + return + try: + # Propagate the signal to our parents iff this is the first field set. + self._parent_message_weakref._Modified() + except ReferenceError: + # We can get here if a client has kept a reference to a child object, + # and is now setting a field on it, but the child's parent has been + # garbage-collected. This is not an error. + pass + + +class _OneofListener(_Listener): + """Special listener implementation for setting composite oneof fields.""" + + def __init__(self, parent_message, field): + """Args: + parent_message: The message whose _Modified() method we should call when + we receive Modified() messages. + field: The descriptor of the field being set in the parent message. + """ + super(_OneofListener, self).__init__(parent_message) + self._field = field + + def Modified(self): + """Also updates the state of the containing oneof in the parent message.""" + try: + self._parent_message_weakref._UpdateOneofState(self._field) + super(_OneofListener, self).Modified() + except ReferenceError: + pass diff --git a/lib/python3.12/site-packages/google/protobuf/internal/testing_refleaks.py b/lib/python3.12/site-packages/google/protobuf/internal/testing_refleaks.py new file mode 100644 index 0000000000000000000000000000000000000000..f94ef2b9d6128b82cc832409843e7d8a57d935c8 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/testing_refleaks.py @@ -0,0 +1,128 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""A subclass of unittest.TestCase which checks for reference leaks. + +To use: +- Use testing_refleak.BaseTestCase instead of unittest.TestCase +- Configure and compile Python with --with-pydebug + +If sys.gettotalrefcount() is not available (because Python was built without +the Py_DEBUG option), then this module is a no-op and tests will run normally. +""" + +import copyreg +import gc +import sys +import unittest + + +class LocalTestResult(unittest.TestResult): + """A TestResult which forwards events to a parent object, except for Skips.""" + + def __init__(self, parent_result): + unittest.TestResult.__init__(self) + self.parent_result = parent_result + + def addError(self, test, error): + self.parent_result.addError(test, error) + + def addFailure(self, test, error): + self.parent_result.addFailure(test, error) + + def addSkip(self, test, reason): + pass + + def addDuration(self, test, duration): + pass + + +class ReferenceLeakCheckerMixin(object): + """A mixin class for TestCase, which checks reference counts.""" + + NB_RUNS = 3 + + def run(self, result=None): + testMethod = getattr(self, self._testMethodName) + expecting_failure_method = getattr(testMethod, "__unittest_expecting_failure__", False) + expecting_failure_class = getattr(self, "__unittest_expecting_failure__", False) + if expecting_failure_class or expecting_failure_method: + return + + # python_message.py registers all Message classes to some pickle global + # registry, which makes the classes immortal. + # We save a copy of this registry, and reset it before we could references. + self._saved_pickle_registry = copyreg.dispatch_table.copy() + + # Run the test twice, to warm up the instance attributes. + super(ReferenceLeakCheckerMixin, self).run(result=result) + super(ReferenceLeakCheckerMixin, self).run(result=result) + + local_result = LocalTestResult(result) + num_flakes = 0 + refcount_deltas = [] + + # Observe the refcount, then create oldrefcount which actually makes the + # refcount 1 higher than the recorded value immediately + oldrefcount = self._getRefcounts() + while len(refcount_deltas) < self.NB_RUNS: + oldrefcount = self._getRefcounts() + super(ReferenceLeakCheckerMixin, self).run(result=local_result) + newrefcount = self._getRefcounts() + # If the GC was able to collect some objects after the call to run() that + # it could not collect before the call, then the counts won't match. + if newrefcount < oldrefcount and num_flakes < 2: + # This result is (probably) a flake -- garbage collectors aren't very + # predictable, but a lower ending refcount is the opposite of the + # failure we are testing for. If the result is repeatable, then we will + # eventually report it, but not after trying to eliminate it. + num_flakes += 1 + continue + num_flakes = 0 + refcount_deltas.append(newrefcount - oldrefcount) + print(refcount_deltas, self) + + try: + self.assertEqual(refcount_deltas, [0] * self.NB_RUNS) + except Exception: # pylint: disable=broad-except + result.addError(self, sys.exc_info()) + + def _getRefcounts(self): + if hasattr(sys, "_clear_internal_caches"): # Since 3.13 + sys._clear_internal_caches() # pylint: disable=protected-access + else: + sys._clear_type_cache() # pylint: disable=protected-access + copyreg.dispatch_table.clear() + copyreg.dispatch_table.update(self._saved_pickle_registry) + # It is sometimes necessary to gc.collect() multiple times, to ensure + # that all objects can be collected. + gc.collect() + gc.collect() + gc.collect() + return sys.gettotalrefcount() + + +if hasattr(sys, 'gettotalrefcount'): + + def TestCase(test_class): + new_bases = (ReferenceLeakCheckerMixin,) + test_class.__bases__ + new_class = type(test_class)( + test_class.__name__, new_bases, dict(test_class.__dict__)) + return new_class + SkipReferenceLeakChecker = unittest.skip + +else: + # When PyDEBUG is not enabled, run the tests normally. + + def TestCase(test_class): + return test_class + + def SkipReferenceLeakChecker(reason): + del reason # Don't skip, so don't need a reason. + def Same(func): + return func + return Same diff --git a/lib/python3.12/site-packages/google/protobuf/internal/type_checkers.py b/lib/python3.12/site-packages/google/protobuf/internal/type_checkers.py new file mode 100644 index 0000000000000000000000000000000000000000..17e814ed9738123af4ece0676b44696479d4833b --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/type_checkers.py @@ -0,0 +1,455 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Provides type checking routines. + +This module defines type checking utilities in the forms of dictionaries: + +VALUE_CHECKERS: A dictionary of field types and a value validation object. +TYPE_TO_BYTE_SIZE_FN: A dictionary with field types and a size computing + function. +TYPE_TO_SERIALIZE_METHOD: A dictionary with field types and serialization + function. +FIELD_TYPE_TO_WIRE_TYPE: A dictionary with field typed and their + corresponding wire types. +TYPE_TO_DESERIALIZE_METHOD: A dictionary with field types and deserialization + function. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import numbers +import struct +import warnings + +from google.protobuf import descriptor +from google.protobuf.internal import decoder +from google.protobuf.internal import encoder +from google.protobuf.internal import wire_format + +_FieldDescriptor = descriptor.FieldDescriptor +# TODO: Remove this warning count after 34.0 +# Assign bool to int/enum warnings will print 100 times at most which should +# be enough for users to notice and do not cause timeout. +_BoolWarningCount = 100 + +def TruncateToFourByteFloat(original): + return struct.unpack(' 0: + _BoolWarningCount -= 1 + message = ( + '%.1024r has type %s, but expected one of: %s. This warning ' + 'will turn into error in 7.34.0, please fix it before that.' + % ( + proposed_value, + type(proposed_value), + (int,), + ) + ) + # TODO: Raise errors in 2026 Q1 release + warnings.warn(message) + + if not hasattr(proposed_value, '__index__') or ( + type(proposed_value).__module__ == 'numpy' and + type(proposed_value).__name__ == 'ndarray'): + message = ('%.1024r has type %s, but expected one of: %s' % + (proposed_value, type(proposed_value), (int,))) + raise TypeError(message) + + if not self._MIN <= int(proposed_value) <= self._MAX: + raise ValueError('Value out of range: %d' % proposed_value) + # We force all values to int to make alternate implementations where the + # distinction is more significant (e.g. the C++ implementation) simpler. + proposed_value = int(proposed_value) + return proposed_value + + def DefaultValue(self): + return 0 + + +class EnumValueChecker(object): + + """Checker used for enum fields. Performs type-check and range check.""" + + def __init__(self, enum_type): + self._enum_type = enum_type + + def CheckValue(self, proposed_value): + global _BoolWarningCount + if type(proposed_value) == bool and _BoolWarningCount > 0: + _BoolWarningCount -= 1 + message = ( + '%.1024r has type %s, but expected one of: %s. This warning ' + 'will turn into error in 7.34.0, please fix it before that.' + % ( + proposed_value, + type(proposed_value), + (int,), + ) + ) + # TODO: Raise errors in 2026 Q1 release + warnings.warn(message) + if not isinstance(proposed_value, numbers.Integral): + message = ('%.1024r has type %s, but expected one of: %s' % + (proposed_value, type(proposed_value), (int,))) + raise TypeError(message) + if int(proposed_value) not in self._enum_type.values_by_number: + raise ValueError('Unknown enum value: %d' % proposed_value) + return proposed_value + + def DefaultValue(self): + return self._enum_type.values[0].number + + +class UnicodeValueChecker(object): + + """Checker used for string fields. + + Always returns a unicode value, even if the input is of type str. + """ + + def CheckValue(self, proposed_value): + if not isinstance(proposed_value, (bytes, str)): + message = ('%.1024r has type %s, but expected one of: %s' % + (proposed_value, type(proposed_value), (bytes, str))) + raise TypeError(message) + + # If the value is of type 'bytes' make sure that it is valid UTF-8 data. + if isinstance(proposed_value, bytes): + try: + proposed_value = proposed_value.decode('utf-8') + except UnicodeDecodeError: + raise ValueError('%.1024r has type bytes, but isn\'t valid UTF-8 ' + 'encoding. Non-UTF-8 strings must be converted to ' + 'unicode objects before being added.' % + (proposed_value)) + else: + try: + proposed_value.encode('utf8') + except UnicodeEncodeError: + raise ValueError('%.1024r isn\'t a valid unicode string and ' + 'can\'t be encoded in UTF-8.'% + (proposed_value)) + + return proposed_value + + def DefaultValue(self): + return u"" + + +class Int32ValueChecker(IntValueChecker): + # We're sure to use ints instead of longs here since comparison may be more + # efficient. + _MIN = -2147483648 + _MAX = 2147483647 + + +class Uint32ValueChecker(IntValueChecker): + _MIN = 0 + _MAX = (1 << 32) - 1 + + +class Int64ValueChecker(IntValueChecker): + _MIN = -(1 << 63) + _MAX = (1 << 63) - 1 + + +class Uint64ValueChecker(IntValueChecker): + _MIN = 0 + _MAX = (1 << 64) - 1 + + +# The max 4 bytes float is about 3.4028234663852886e+38 +_FLOAT_MAX = float.fromhex('0x1.fffffep+127') +_FLOAT_MIN = -_FLOAT_MAX +_MAX_FLOAT_AS_DOUBLE_ROUNDED = 3.4028235677973366e38 +_INF = float('inf') +_NEG_INF = float('-inf') + + +class DoubleValueChecker(object): + """Checker used for double fields. + + Performs type-check and range check. + """ + + def CheckValue(self, proposed_value): + """Check and convert proposed_value to float.""" + if (not hasattr(proposed_value, '__float__') and + not hasattr(proposed_value, '__index__')) or ( + type(proposed_value).__module__ == 'numpy' and + type(proposed_value).__name__ == 'ndarray'): + message = ('%.1024r has type %s, but expected one of: int, float' % + (proposed_value, type(proposed_value))) + raise TypeError(message) + return float(proposed_value) + + def DefaultValue(self): + return 0.0 + + +class FloatValueChecker(DoubleValueChecker): + """Checker used for float fields. + + Performs type-check and range check. + + Values exceeding a 32-bit float will be converted to inf/-inf. + """ + + def CheckValue(self, proposed_value): + """Check and convert proposed_value to float.""" + converted_value = super().CheckValue(proposed_value) + # This inf rounding matches the C++ proto SafeDoubleToFloat logic. + if converted_value > _FLOAT_MAX: + if converted_value <= _MAX_FLOAT_AS_DOUBLE_ROUNDED: + return _FLOAT_MAX + return _INF + if converted_value < _FLOAT_MIN: + if converted_value >= -_MAX_FLOAT_AS_DOUBLE_ROUNDED: + return _FLOAT_MIN + return _NEG_INF + + return TruncateToFourByteFloat(converted_value) + +# Type-checkers for all scalar CPPTYPEs. +_VALUE_CHECKERS = { + _FieldDescriptor.CPPTYPE_INT32: Int32ValueChecker(), + _FieldDescriptor.CPPTYPE_INT64: Int64ValueChecker(), + _FieldDescriptor.CPPTYPE_UINT32: Uint32ValueChecker(), + _FieldDescriptor.CPPTYPE_UINT64: Uint64ValueChecker(), + _FieldDescriptor.CPPTYPE_DOUBLE: DoubleValueChecker(), + _FieldDescriptor.CPPTYPE_FLOAT: FloatValueChecker(), + _FieldDescriptor.CPPTYPE_BOOL: BoolValueChecker(), + _FieldDescriptor.CPPTYPE_STRING: TypeCheckerWithDefault(b'', bytes), +} + + +# Map from field type to a function F, such that F(field_num, value) +# gives the total byte size for a value of the given type. This +# byte size includes tag information and any other additional space +# associated with serializing "value". +TYPE_TO_BYTE_SIZE_FN = { + _FieldDescriptor.TYPE_DOUBLE: wire_format.DoubleByteSize, + _FieldDescriptor.TYPE_FLOAT: wire_format.FloatByteSize, + _FieldDescriptor.TYPE_INT64: wire_format.Int64ByteSize, + _FieldDescriptor.TYPE_UINT64: wire_format.UInt64ByteSize, + _FieldDescriptor.TYPE_INT32: wire_format.Int32ByteSize, + _FieldDescriptor.TYPE_FIXED64: wire_format.Fixed64ByteSize, + _FieldDescriptor.TYPE_FIXED32: wire_format.Fixed32ByteSize, + _FieldDescriptor.TYPE_BOOL: wire_format.BoolByteSize, + _FieldDescriptor.TYPE_STRING: wire_format.StringByteSize, + _FieldDescriptor.TYPE_GROUP: wire_format.GroupByteSize, + _FieldDescriptor.TYPE_MESSAGE: wire_format.MessageByteSize, + _FieldDescriptor.TYPE_BYTES: wire_format.BytesByteSize, + _FieldDescriptor.TYPE_UINT32: wire_format.UInt32ByteSize, + _FieldDescriptor.TYPE_ENUM: wire_format.EnumByteSize, + _FieldDescriptor.TYPE_SFIXED32: wire_format.SFixed32ByteSize, + _FieldDescriptor.TYPE_SFIXED64: wire_format.SFixed64ByteSize, + _FieldDescriptor.TYPE_SINT32: wire_format.SInt32ByteSize, + _FieldDescriptor.TYPE_SINT64: wire_format.SInt64ByteSize + } + + +# Maps from field types to encoder constructors. +TYPE_TO_ENCODER = { + _FieldDescriptor.TYPE_DOUBLE: encoder.DoubleEncoder, + _FieldDescriptor.TYPE_FLOAT: encoder.FloatEncoder, + _FieldDescriptor.TYPE_INT64: encoder.Int64Encoder, + _FieldDescriptor.TYPE_UINT64: encoder.UInt64Encoder, + _FieldDescriptor.TYPE_INT32: encoder.Int32Encoder, + _FieldDescriptor.TYPE_FIXED64: encoder.Fixed64Encoder, + _FieldDescriptor.TYPE_FIXED32: encoder.Fixed32Encoder, + _FieldDescriptor.TYPE_BOOL: encoder.BoolEncoder, + _FieldDescriptor.TYPE_STRING: encoder.StringEncoder, + _FieldDescriptor.TYPE_GROUP: encoder.GroupEncoder, + _FieldDescriptor.TYPE_MESSAGE: encoder.MessageEncoder, + _FieldDescriptor.TYPE_BYTES: encoder.BytesEncoder, + _FieldDescriptor.TYPE_UINT32: encoder.UInt32Encoder, + _FieldDescriptor.TYPE_ENUM: encoder.EnumEncoder, + _FieldDescriptor.TYPE_SFIXED32: encoder.SFixed32Encoder, + _FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Encoder, + _FieldDescriptor.TYPE_SINT32: encoder.SInt32Encoder, + _FieldDescriptor.TYPE_SINT64: encoder.SInt64Encoder, + } + + +# Maps from field types to sizer constructors. +TYPE_TO_SIZER = { + _FieldDescriptor.TYPE_DOUBLE: encoder.DoubleSizer, + _FieldDescriptor.TYPE_FLOAT: encoder.FloatSizer, + _FieldDescriptor.TYPE_INT64: encoder.Int64Sizer, + _FieldDescriptor.TYPE_UINT64: encoder.UInt64Sizer, + _FieldDescriptor.TYPE_INT32: encoder.Int32Sizer, + _FieldDescriptor.TYPE_FIXED64: encoder.Fixed64Sizer, + _FieldDescriptor.TYPE_FIXED32: encoder.Fixed32Sizer, + _FieldDescriptor.TYPE_BOOL: encoder.BoolSizer, + _FieldDescriptor.TYPE_STRING: encoder.StringSizer, + _FieldDescriptor.TYPE_GROUP: encoder.GroupSizer, + _FieldDescriptor.TYPE_MESSAGE: encoder.MessageSizer, + _FieldDescriptor.TYPE_BYTES: encoder.BytesSizer, + _FieldDescriptor.TYPE_UINT32: encoder.UInt32Sizer, + _FieldDescriptor.TYPE_ENUM: encoder.EnumSizer, + _FieldDescriptor.TYPE_SFIXED32: encoder.SFixed32Sizer, + _FieldDescriptor.TYPE_SFIXED64: encoder.SFixed64Sizer, + _FieldDescriptor.TYPE_SINT32: encoder.SInt32Sizer, + _FieldDescriptor.TYPE_SINT64: encoder.SInt64Sizer, + } + + +# Maps from field type to a decoder constructor. +TYPE_TO_DECODER = { + _FieldDescriptor.TYPE_DOUBLE: decoder.DoubleDecoder, + _FieldDescriptor.TYPE_FLOAT: decoder.FloatDecoder, + _FieldDescriptor.TYPE_INT64: decoder.Int64Decoder, + _FieldDescriptor.TYPE_UINT64: decoder.UInt64Decoder, + _FieldDescriptor.TYPE_INT32: decoder.Int32Decoder, + _FieldDescriptor.TYPE_FIXED64: decoder.Fixed64Decoder, + _FieldDescriptor.TYPE_FIXED32: decoder.Fixed32Decoder, + _FieldDescriptor.TYPE_BOOL: decoder.BoolDecoder, + _FieldDescriptor.TYPE_STRING: decoder.StringDecoder, + _FieldDescriptor.TYPE_GROUP: decoder.GroupDecoder, + _FieldDescriptor.TYPE_MESSAGE: decoder.MessageDecoder, + _FieldDescriptor.TYPE_BYTES: decoder.BytesDecoder, + _FieldDescriptor.TYPE_UINT32: decoder.UInt32Decoder, + _FieldDescriptor.TYPE_ENUM: decoder.EnumDecoder, + _FieldDescriptor.TYPE_SFIXED32: decoder.SFixed32Decoder, + _FieldDescriptor.TYPE_SFIXED64: decoder.SFixed64Decoder, + _FieldDescriptor.TYPE_SINT32: decoder.SInt32Decoder, + _FieldDescriptor.TYPE_SINT64: decoder.SInt64Decoder, + } + +# Maps from field type to expected wiretype. +FIELD_TYPE_TO_WIRE_TYPE = { + _FieldDescriptor.TYPE_DOUBLE: wire_format.WIRETYPE_FIXED64, + _FieldDescriptor.TYPE_FLOAT: wire_format.WIRETYPE_FIXED32, + _FieldDescriptor.TYPE_INT64: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_UINT64: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_INT32: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_FIXED64: wire_format.WIRETYPE_FIXED64, + _FieldDescriptor.TYPE_FIXED32: wire_format.WIRETYPE_FIXED32, + _FieldDescriptor.TYPE_BOOL: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_STRING: + wire_format.WIRETYPE_LENGTH_DELIMITED, + _FieldDescriptor.TYPE_GROUP: wire_format.WIRETYPE_START_GROUP, + _FieldDescriptor.TYPE_MESSAGE: + wire_format.WIRETYPE_LENGTH_DELIMITED, + _FieldDescriptor.TYPE_BYTES: + wire_format.WIRETYPE_LENGTH_DELIMITED, + _FieldDescriptor.TYPE_UINT32: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_ENUM: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_SFIXED32: wire_format.WIRETYPE_FIXED32, + _FieldDescriptor.TYPE_SFIXED64: wire_format.WIRETYPE_FIXED64, + _FieldDescriptor.TYPE_SINT32: wire_format.WIRETYPE_VARINT, + _FieldDescriptor.TYPE_SINT64: wire_format.WIRETYPE_VARINT, + } diff --git a/lib/python3.12/site-packages/google/protobuf/internal/well_known_types.py b/lib/python3.12/site-packages/google/protobuf/internal/well_known_types.py new file mode 100644 index 0000000000000000000000000000000000000000..32b625a3022e4f30b820b1d24ac15ffefb9b9c93 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/well_known_types.py @@ -0,0 +1,695 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains well known classes. + +This files defines well known classes which need extra maintenance including: + - Any + - Duration + - FieldMask + - Struct + - Timestamp +""" + +__author__ = 'jieluo@google.com (Jie Luo)' + +import calendar +import collections.abc +import datetime +from typing import Union +import warnings +from google.protobuf.internal import field_mask + +FieldMask = field_mask.FieldMask + +_TIMESTAMPFORMAT = '%Y-%m-%dT%H:%M:%S' +_NANOS_PER_SECOND = 1000000000 +_NANOS_PER_MILLISECOND = 1000000 +_NANOS_PER_MICROSECOND = 1000 +_MILLIS_PER_SECOND = 1000 +_MICROS_PER_SECOND = 1000000 +_SECONDS_PER_DAY = 24 * 3600 +_DURATION_SECONDS_MAX = 315576000000 +_TIMESTAMP_SECONDS_MIN = -62135596800 +_TIMESTAMP_SECONDS_MAX = 253402300799 + +_EPOCH_DATETIME_NAIVE = datetime.datetime(1970, 1, 1, tzinfo=None) +_EPOCH_DATETIME_AWARE = _EPOCH_DATETIME_NAIVE.replace( + tzinfo=datetime.timezone.utc +) + + +class Any(object): + """Class for Any Message type.""" + + __slots__ = () + + def Pack( + self, msg, type_url_prefix='type.googleapis.com/', deterministic=None + ): + """Packs the specified message into current Any message.""" + if len(type_url_prefix) < 1 or type_url_prefix[-1] != '/': + self.type_url = '%s/%s' % (type_url_prefix, msg.DESCRIPTOR.full_name) + else: + self.type_url = '%s%s' % (type_url_prefix, msg.DESCRIPTOR.full_name) + self.value = msg.SerializeToString(deterministic=deterministic) + + def Unpack(self, msg): + """Unpacks the current Any message into specified message.""" + descriptor = msg.DESCRIPTOR + if not self.Is(descriptor): + return False + msg.ParseFromString(self.value) + return True + + def TypeName(self): + """Returns the protobuf type name of the inner message.""" + # Only last part is to be used: b/25630112 + return self.type_url.rpartition('/')[2] + + def Is(self, descriptor): + """Checks if this Any represents the given protobuf type.""" + return '/' in self.type_url and self.TypeName() == descriptor.full_name + + +class Timestamp(object): + """Class for Timestamp message type.""" + + __slots__ = () + + def ToJsonString(self): + """Converts Timestamp to RFC 3339 date string format. + + Returns: + A string converted from timestamp. The string is always Z-normalized + and uses 3, 6 or 9 fractional digits as required to represent the + exact time. Example of the return format: '1972-01-01T10:00:20.021Z' + """ + _CheckTimestampValid(self.seconds, self.nanos) + nanos = self.nanos + seconds = self.seconds % _SECONDS_PER_DAY + days = (self.seconds - seconds) // _SECONDS_PER_DAY + dt = datetime.datetime(1970, 1, 1) + datetime.timedelta(days, seconds) + + result = dt.isoformat() + if (nanos % 1e9) == 0: + # If there are 0 fractional digits, the fractional + # point '.' should be omitted when serializing. + return result + 'Z' + if (nanos % 1e6) == 0: + # Serialize 3 fractional digits. + return result + '.%03dZ' % (nanos / 1e6) + if (nanos % 1e3) == 0: + # Serialize 6 fractional digits. + return result + '.%06dZ' % (nanos / 1e3) + # Serialize 9 fractional digits. + return result + '.%09dZ' % nanos + + def FromJsonString(self, value): + """Parse a RFC 3339 date string format to Timestamp. + + Args: + value: A date string. Any fractional digits (or none) and any offset are + accepted as long as they fit into nano-seconds precision. Example of + accepted format: '1972-01-01T10:00:20.021-05:00' + + Raises: + ValueError: On parsing problems. + """ + if not isinstance(value, str): + raise ValueError('Timestamp JSON value not a string: {!r}'.format(value)) + timezone_offset = value.find('Z') + if timezone_offset == -1: + timezone_offset = value.find('+') + if timezone_offset == -1: + timezone_offset = value.rfind('-') + if timezone_offset == -1: + raise ValueError( + 'Failed to parse timestamp: missing valid timezone offset.' + ) + time_value = value[0:timezone_offset] + # Parse datetime and nanos. + point_position = time_value.find('.') + if point_position == -1: + second_value = time_value + nano_value = '' + else: + second_value = time_value[:point_position] + nano_value = time_value[point_position + 1 :] + if 't' in second_value: + raise ValueError( + "time data '{0}' does not match format '%Y-%m-%dT%H:%M:%S', " + "lowercase 't' is not accepted".format(second_value) + ) + date_object = datetime.datetime.strptime(second_value, _TIMESTAMPFORMAT) + td = date_object - datetime.datetime(1970, 1, 1) + seconds = td.seconds + td.days * _SECONDS_PER_DAY + if len(nano_value) > 9: + raise ValueError( + 'Failed to parse Timestamp: nanos {0} more than ' + '9 fractional digits.'.format(nano_value) + ) + if nano_value: + nanos = round(float('0.' + nano_value) * 1e9) + else: + nanos = 0 + # Parse timezone offsets. + if value[timezone_offset] == 'Z': + if len(value) != timezone_offset + 1: + raise ValueError( + 'Failed to parse timestamp: invalid trailing data {0}.'.format( + value + ) + ) + else: + timezone = value[timezone_offset:] + pos = timezone.find(':') + if pos == -1: + raise ValueError('Invalid timezone offset value: {0}.'.format(timezone)) + if timezone[0] == '+': + seconds -= (int(timezone[1:pos]) * 60 + int(timezone[pos + 1 :])) * 60 + else: + seconds += (int(timezone[1:pos]) * 60 + int(timezone[pos + 1 :])) * 60 + # Set seconds and nanos + _CheckTimestampValid(seconds, nanos) + self.seconds = int(seconds) + self.nanos = int(nanos) + + def GetCurrentTime(self): + """Get the current UTC into Timestamp.""" + self.FromDatetime(datetime.datetime.now(tz=datetime.timezone.utc)) + + def ToNanoseconds(self): + """Converts Timestamp to nanoseconds since epoch.""" + _CheckTimestampValid(self.seconds, self.nanos) + return self.seconds * _NANOS_PER_SECOND + self.nanos + + def ToMicroseconds(self): + """Converts Timestamp to microseconds since epoch.""" + _CheckTimestampValid(self.seconds, self.nanos) + return ( + self.seconds * _MICROS_PER_SECOND + self.nanos // _NANOS_PER_MICROSECOND + ) + + def ToMilliseconds(self): + """Converts Timestamp to milliseconds since epoch.""" + _CheckTimestampValid(self.seconds, self.nanos) + return ( + self.seconds * _MILLIS_PER_SECOND + self.nanos // _NANOS_PER_MILLISECOND + ) + + def ToSeconds(self): + """Converts Timestamp to seconds since epoch.""" + _CheckTimestampValid(self.seconds, self.nanos) + return self.seconds + + def FromNanoseconds(self, nanos): + """Converts nanoseconds since epoch to Timestamp.""" + seconds = nanos // _NANOS_PER_SECOND + nanos = nanos % _NANOS_PER_SECOND + _CheckTimestampValid(seconds, nanos) + self.seconds = seconds + self.nanos = nanos + + def FromMicroseconds(self, micros): + """Converts microseconds since epoch to Timestamp.""" + seconds = micros // _MICROS_PER_SECOND + nanos = (micros % _MICROS_PER_SECOND) * _NANOS_PER_MICROSECOND + _CheckTimestampValid(seconds, nanos) + self.seconds = seconds + self.nanos = nanos + + def FromMilliseconds(self, millis): + """Converts milliseconds since epoch to Timestamp.""" + seconds = millis // _MILLIS_PER_SECOND + nanos = (millis % _MILLIS_PER_SECOND) * _NANOS_PER_MILLISECOND + _CheckTimestampValid(seconds, nanos) + self.seconds = seconds + self.nanos = nanos + + def FromSeconds(self, seconds): + """Converts seconds since epoch to Timestamp.""" + _CheckTimestampValid(seconds, 0) + self.seconds = seconds + self.nanos = 0 + + def ToDatetime(self, tzinfo=None): + """Converts Timestamp to a datetime. + + Args: + tzinfo: A datetime.tzinfo subclass; defaults to None. + + Returns: + If tzinfo is None, returns a timezone-naive UTC datetime (with no timezone + information, i.e. not aware that it's UTC). + + Otherwise, returns a timezone-aware datetime in the input timezone. + """ + # Using datetime.fromtimestamp for this would avoid constructing an extra + # timedelta object and possibly an extra datetime. Unfortunately, that has + # the disadvantage of not handling the full precision (on all platforms, see + # https://github.com/python/cpython/issues/109849) or full range (on some + # platforms, see https://github.com/python/cpython/issues/110042) of + # datetime. + _CheckTimestampValid(self.seconds, self.nanos) + delta = datetime.timedelta( + seconds=self.seconds, + microseconds=_RoundTowardZero(self.nanos, _NANOS_PER_MICROSECOND), + ) + if tzinfo is None: + return _EPOCH_DATETIME_NAIVE + delta + else: + # Note the tz conversion has to come after the timedelta arithmetic. + return (_EPOCH_DATETIME_AWARE + delta).astimezone(tzinfo) + + def FromDatetime(self, dt): + """Converts datetime to Timestamp. + + Args: + dt: A datetime. If it's timezone-naive, it's assumed to be in UTC. + """ + # Using this guide: http://wiki.python.org/moin/WorkingWithTime + # And this conversion guide: http://docs.python.org/library/time.html + + # Turn the date parameter into a tuple (struct_time) that can then be + # manipulated into a long value of seconds. During the conversion from + # struct_time to long, the source date in UTC, and so it follows that the + # correct transformation is calendar.timegm() + try: + seconds = calendar.timegm(dt.utctimetuple()) + nanos = dt.microsecond * _NANOS_PER_MICROSECOND + except AttributeError as e: + raise AttributeError( + 'Fail to convert to Timestamp. Expected a datetime like ' + 'object got {0} : {1}'.format(type(dt).__name__, e) + ) from e + _CheckTimestampValid(seconds, nanos) + self.seconds = seconds + self.nanos = nanos + + def _internal_assign(self, dt): + self.FromDatetime(dt) + + def __add__(self, value) -> datetime.datetime: + if isinstance(value, Duration): + return self.ToDatetime() + value.ToTimedelta() + return self.ToDatetime() + value + + __radd__ = __add__ + + def __sub__(self, value) -> Union[datetime.datetime, datetime.timedelta]: + if isinstance(value, Timestamp): + return self.ToDatetime() - value.ToDatetime() + elif isinstance(value, Duration): + return self.ToDatetime() - value.ToTimedelta() + return self.ToDatetime() - value + + def __rsub__(self, dt) -> datetime.timedelta: + return dt - self.ToDatetime() + + +def _CheckTimestampValid(seconds, nanos): + if seconds < _TIMESTAMP_SECONDS_MIN or seconds > _TIMESTAMP_SECONDS_MAX: + raise ValueError( + 'Timestamp is not valid: Seconds {0} must be in range ' + '[-62135596800, 253402300799].'.format(seconds)) + if nanos < 0 or nanos >= _NANOS_PER_SECOND: + raise ValueError( + 'Timestamp is not valid: Nanos {} must be in a range ' + '[0, 999999].'.format(nanos) + ) + + +class Duration(object): + """Class for Duration message type.""" + + __slots__ = () + + def ToJsonString(self): + """Converts Duration to string format. + + Returns: + A string converted from self. The string format will contains + 3, 6, or 9 fractional digits depending on the precision required to + represent the exact Duration value. For example: "1s", "1.010s", + "1.000000100s", "-3.100s" + """ + _CheckDurationValid(self.seconds, self.nanos) + if self.seconds < 0 or self.nanos < 0: + result = '-' + seconds = -self.seconds + int((0 - self.nanos) // 1e9) + nanos = (0 - self.nanos) % 1e9 + else: + result = '' + seconds = self.seconds + int(self.nanos // 1e9) + nanos = self.nanos % 1e9 + result += '%d' % seconds + if (nanos % 1e9) == 0: + # If there are 0 fractional digits, the fractional + # point '.' should be omitted when serializing. + return result + 's' + if (nanos % 1e6) == 0: + # Serialize 3 fractional digits. + return result + '.%03ds' % (nanos / 1e6) + if (nanos % 1e3) == 0: + # Serialize 6 fractional digits. + return result + '.%06ds' % (nanos / 1e3) + # Serialize 9 fractional digits. + return result + '.%09ds' % nanos + + def FromJsonString(self, value): + """Converts a string to Duration. + + Args: + value: A string to be converted. The string must end with 's'. Any + fractional digits (or none) are accepted as long as they fit into + precision. For example: "1s", "1.01s", "1.0000001s", "-3.100s + + Raises: + ValueError: On parsing problems. + """ + if not isinstance(value, str): + raise ValueError('Duration JSON value not a string: {!r}'.format(value)) + if len(value) < 1 or value[-1] != 's': + raise ValueError('Duration must end with letter "s": {0}.'.format(value)) + try: + pos = value.find('.') + if pos == -1: + seconds = int(value[:-1]) + nanos = 0 + else: + seconds = int(value[:pos]) + if value[0] == '-': + nanos = int(round(float('-0{0}'.format(value[pos:-1])) * 1e9)) + else: + nanos = int(round(float('0{0}'.format(value[pos:-1])) * 1e9)) + _CheckDurationValid(seconds, nanos) + self.seconds = seconds + self.nanos = nanos + except ValueError as e: + raise ValueError("Couldn't parse duration: {0} : {1}.".format(value, e)) + + def ToNanoseconds(self): + """Converts a Duration to nanoseconds.""" + return self.seconds * _NANOS_PER_SECOND + self.nanos + + def ToMicroseconds(self): + """Converts a Duration to microseconds.""" + micros = _RoundTowardZero(self.nanos, _NANOS_PER_MICROSECOND) + return self.seconds * _MICROS_PER_SECOND + micros + + def ToMilliseconds(self): + """Converts a Duration to milliseconds.""" + millis = _RoundTowardZero(self.nanos, _NANOS_PER_MILLISECOND) + return self.seconds * _MILLIS_PER_SECOND + millis + + def ToSeconds(self): + """Converts a Duration to seconds.""" + return self.seconds + + def FromNanoseconds(self, nanos): + """Converts nanoseconds to Duration.""" + self._NormalizeDuration( + nanos // _NANOS_PER_SECOND, nanos % _NANOS_PER_SECOND + ) + + def FromMicroseconds(self, micros): + """Converts microseconds to Duration.""" + self._NormalizeDuration( + micros // _MICROS_PER_SECOND, + (micros % _MICROS_PER_SECOND) * _NANOS_PER_MICROSECOND, + ) + + def FromMilliseconds(self, millis): + """Converts milliseconds to Duration.""" + self._NormalizeDuration( + millis // _MILLIS_PER_SECOND, + (millis % _MILLIS_PER_SECOND) * _NANOS_PER_MILLISECOND, + ) + + def FromSeconds(self, seconds): + """Converts seconds to Duration.""" + self.seconds = seconds + self.nanos = 0 + + def ToTimedelta(self) -> datetime.timedelta: + """Converts Duration to timedelta.""" + return datetime.timedelta( + seconds=self.seconds, + microseconds=_RoundTowardZero(self.nanos, _NANOS_PER_MICROSECOND), + ) + + def FromTimedelta(self, td): + """Converts timedelta to Duration.""" + try: + self._NormalizeDuration( + td.seconds + td.days * _SECONDS_PER_DAY, + td.microseconds * _NANOS_PER_MICROSECOND, + ) + except AttributeError as e: + raise AttributeError( + 'Fail to convert to Duration. Expected a timedelta like ' + 'object got {0}: {1}'.format(type(td).__name__, e) + ) from e + + def _internal_assign(self, td): + self.FromTimedelta(td) + + def _NormalizeDuration(self, seconds, nanos): + """Set Duration by seconds and nanos.""" + # Force nanos to be negative if the duration is negative. + if seconds < 0 and nanos > 0: + seconds += 1 + nanos -= _NANOS_PER_SECOND + self.seconds = seconds + self.nanos = nanos + + def __add__(self, value) -> Union[datetime.datetime, datetime.timedelta]: + if isinstance(value, Timestamp): + return self.ToTimedelta() + value.ToDatetime() + return self.ToTimedelta() + value + + __radd__ = __add__ + + def __sub__(self, value) -> datetime.timedelta: + return self.ToTimedelta() - value + + def __rsub__(self, value) -> Union[datetime.datetime, datetime.timedelta]: + return value - self.ToTimedelta() + + +def _CheckDurationValid(seconds, nanos): + if seconds < -_DURATION_SECONDS_MAX or seconds > _DURATION_SECONDS_MAX: + raise ValueError( + 'Duration is not valid: Seconds {0} must be in range ' + '[-315576000000, 315576000000].'.format(seconds) + ) + if nanos <= -_NANOS_PER_SECOND or nanos >= _NANOS_PER_SECOND: + raise ValueError( + 'Duration is not valid: Nanos {0} must be in range ' + '[-999999999, 999999999].'.format(nanos) + ) + if (nanos < 0 and seconds > 0) or (nanos > 0 and seconds < 0): + raise ValueError('Duration is not valid: Sign mismatch.') + + +def _RoundTowardZero(value, divider): + """Truncates the remainder part after division.""" + # For some languages, the sign of the remainder is implementation + # dependent if any of the operands is negative. Here we enforce + # "rounded toward zero" semantics. For example, for (-5) / 2 an + # implementation may give -3 as the result with the remainder being + # 1. This function ensures we always return -2 (closer to zero). + result = value // divider + remainder = value % divider + if result < 0 and remainder > 0: + return result + 1 + else: + return result + + +def _SetStructValue(struct_value, value): + if value is None: + struct_value.null_value = 0 + elif isinstance(value, bool): + # Note: this check must come before the number check because in Python + # True and False are also considered numbers. + struct_value.bool_value = value + elif isinstance(value, str): + struct_value.string_value = value + elif isinstance(value, (int, float)): + struct_value.number_value = value + elif isinstance(value, (dict, Struct)): + struct_value.struct_value.Clear() + struct_value.struct_value.update(value) + elif isinstance(value, (list, tuple, ListValue)): + struct_value.list_value.Clear() + struct_value.list_value.extend(value) + else: + raise ValueError('Unexpected type') + + +def _GetStructValue(struct_value): + which = struct_value.WhichOneof('kind') + if which == 'struct_value': + return struct_value.struct_value + elif which == 'null_value': + return None + elif which == 'number_value': + return struct_value.number_value + elif which == 'string_value': + return struct_value.string_value + elif which == 'bool_value': + return struct_value.bool_value + elif which == 'list_value': + return struct_value.list_value + elif which is None: + raise ValueError('Value not set') + + +class Struct(object): + """Class for Struct message type.""" + + __slots__ = () + + def __getitem__(self, key): + return _GetStructValue(self.fields[key]) + + def __setitem__(self, key, value): + _SetStructValue(self.fields[key], value) + + def __delitem__(self, key): + del self.fields[key] + + def __len__(self): + return len(self.fields) + + def __iter__(self): + return iter(self.fields) + + def _internal_assign(self, dictionary): + self.Clear() + self.update(dictionary) + + def _internal_compare(self, other): + size = len(self) + if size != len(other): + return False + for key, value in self.items(): + if key not in other: + return False + if isinstance(other[key], (dict, list)): + if not value._internal_compare(other[key]): + return False + elif value != other[key]: + return False + return True + + def keys(self): # pylint: disable=invalid-name + return self.fields.keys() + + def values(self): # pylint: disable=invalid-name + return [self[key] for key in self] + + def items(self): # pylint: disable=invalid-name + return [(key, self[key]) for key in self] + + def get_or_create_list(self, key): + """Returns a list for this key, creating if it didn't exist already.""" + if not self.fields[key].HasField('list_value'): + # Clear will mark list_value modified which will indeed create a list. + self.fields[key].list_value.Clear() + return self.fields[key].list_value + + def get_or_create_struct(self, key): + """Returns a struct for this key, creating if it didn't exist already.""" + if not self.fields[key].HasField('struct_value'): + # Clear will mark struct_value modified which will indeed create a struct. + self.fields[key].struct_value.Clear() + return self.fields[key].struct_value + + def update(self, dictionary): # pylint: disable=invalid-name + for key, value in dictionary.items(): + _SetStructValue(self.fields[key], value) + + +collections.abc.MutableMapping.register(Struct) + + +class ListValue(object): + """Class for ListValue message type.""" + + __slots__ = () + + def __len__(self): + return len(self.values) + + def append(self, value): + _SetStructValue(self.values.add(), value) + + def extend(self, elem_seq): + for value in elem_seq: + self.append(value) + + def __getitem__(self, index): + """Retrieves item by the specified index.""" + return _GetStructValue(self.values.__getitem__(index)) + + def __setitem__(self, index, value): + _SetStructValue(self.values.__getitem__(index), value) + + def __delitem__(self, key): + del self.values[key] + + def _internal_assign(self, elem_seq): + self.Clear() + self.extend(elem_seq) + + def _internal_compare(self, other): + size = len(self) + if size != len(other): + return False + for i in range(size): + if isinstance(other[i], (dict, list)): + if not self[i]._internal_compare(other[i]): + return False + elif self[i] != other[i]: + return False + return True + + def items(self): + for i in range(len(self)): + yield self[i] + + def add_struct(self): + """Appends and returns a struct value as the next value in the list.""" + struct_value = self.values.add().struct_value + # Clear will mark struct_value modified which will indeed create a struct. + struct_value.Clear() + return struct_value + + def add_list(self): + """Appends and returns a list value as the next value in the list.""" + list_value = self.values.add().list_value + # Clear will mark list_value modified which will indeed create a list. + list_value.Clear() + return list_value + + +collections.abc.MutableSequence.register(ListValue) + + +# LINT.IfChange(wktbases) +WKTBASES = { + 'google.protobuf.Any': Any, + 'google.protobuf.Duration': Duration, + 'google.protobuf.FieldMask': FieldMask, + 'google.protobuf.ListValue': ListValue, + 'google.protobuf.Struct': Struct, + 'google.protobuf.Timestamp': Timestamp, +} +# LINT.ThenChange(//depot/google.protobuf/compiler/python/pyi_generator.cc:wktbases) diff --git a/lib/python3.12/site-packages/google/protobuf/internal/wire_format.py b/lib/python3.12/site-packages/google/protobuf/internal/wire_format.py new file mode 100644 index 0000000000000000000000000000000000000000..6237dabf950e49d2d6124340c06724f41c1c2c8d --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/internal/wire_format.py @@ -0,0 +1,245 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Constants and static functions to support protocol buffer wire format.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import struct +from google.protobuf import descriptor +from google.protobuf import message + + +TAG_TYPE_BITS = 3 # Number of bits used to hold type info in a proto tag. +TAG_TYPE_MASK = (1 << TAG_TYPE_BITS) - 1 # 0x7 + +# These numbers identify the wire type of a protocol buffer value. +# We use the least-significant TAG_TYPE_BITS bits of the varint-encoded +# tag-and-type to store one of these WIRETYPE_* constants. +# These values must match WireType enum in //google/protobuf/wire_format.h. +WIRETYPE_VARINT = 0 +WIRETYPE_FIXED64 = 1 +WIRETYPE_LENGTH_DELIMITED = 2 +WIRETYPE_START_GROUP = 3 +WIRETYPE_END_GROUP = 4 +WIRETYPE_FIXED32 = 5 +_WIRETYPE_MAX = 5 + + +# Bounds for various integer types. +INT32_MAX = int((1 << 31) - 1) +INT32_MIN = int(-(1 << 31)) +UINT32_MAX = (1 << 32) - 1 + +INT64_MAX = (1 << 63) - 1 +INT64_MIN = -(1 << 63) +UINT64_MAX = (1 << 64) - 1 + +# "struct" format strings that will encode/decode the specified formats. +FORMAT_UINT32_LITTLE_ENDIAN = '> TAG_TYPE_BITS), (tag & TAG_TYPE_MASK) + + +def ZigZagEncode(value): + """ZigZag Transform: Encodes signed integers so that they can be + effectively used with varint encoding. See wire_format.h for + more details. + """ + if value >= 0: + return value << 1 + return (value << 1) ^ (~0) + + +def ZigZagDecode(value): + """Inverse of ZigZagEncode().""" + if not value & 0x1: + return value >> 1 + return (value >> 1) ^ (~0) + + + +# The *ByteSize() functions below return the number of bytes required to +# serialize "field number + type" information and then serialize the value. + + +def Int32ByteSize(field_number, int32): + return Int64ByteSize(field_number, int32) + + +def Int32ByteSizeNoTag(int32): + return _VarUInt64ByteSizeNoTag(0xffffffffffffffff & int32) + + +def Int64ByteSize(field_number, int64): + # Have to convert to uint before calling UInt64ByteSize(). + return UInt64ByteSize(field_number, 0xffffffffffffffff & int64) + + +def UInt32ByteSize(field_number, uint32): + return UInt64ByteSize(field_number, uint32) + + +def UInt64ByteSize(field_number, uint64): + return TagByteSize(field_number) + _VarUInt64ByteSizeNoTag(uint64) + + +def SInt32ByteSize(field_number, int32): + return UInt32ByteSize(field_number, ZigZagEncode(int32)) + + +def SInt64ByteSize(field_number, int64): + return UInt64ByteSize(field_number, ZigZagEncode(int64)) + + +def Fixed32ByteSize(field_number, fixed32): + return TagByteSize(field_number) + 4 + + +def Fixed64ByteSize(field_number, fixed64): + return TagByteSize(field_number) + 8 + + +def SFixed32ByteSize(field_number, sfixed32): + return TagByteSize(field_number) + 4 + + +def SFixed64ByteSize(field_number, sfixed64): + return TagByteSize(field_number) + 8 + + +def FloatByteSize(field_number, flt): + return TagByteSize(field_number) + 4 + + +def DoubleByteSize(field_number, double): + return TagByteSize(field_number) + 8 + + +def BoolByteSize(field_number, b): + return TagByteSize(field_number) + 1 + + +def EnumByteSize(field_number, enum): + return UInt32ByteSize(field_number, enum) + + +def StringByteSize(field_number, string): + return BytesByteSize(field_number, string.encode('utf-8')) + + +def BytesByteSize(field_number, b): + return (TagByteSize(field_number) + + _VarUInt64ByteSizeNoTag(len(b)) + + len(b)) + + +def GroupByteSize(field_number, message): + return (2 * TagByteSize(field_number) # START and END group. + + message.ByteSize()) + + +def MessageByteSize(field_number, message): + return (TagByteSize(field_number) + + _VarUInt64ByteSizeNoTag(message.ByteSize()) + + message.ByteSize()) + + +def MessageSetItemByteSize(field_number, msg): + # First compute the sizes of the tags. + # There are 2 tags for the beginning and ending of the repeated group, that + # is field number 1, one with field number 2 (type_id) and one with field + # number 3 (message). + total_size = (2 * TagByteSize(1) + TagByteSize(2) + TagByteSize(3)) + + # Add the number of bytes for type_id. + total_size += _VarUInt64ByteSizeNoTag(field_number) + + message_size = msg.ByteSize() + + # The number of bytes for encoding the length of the message. + total_size += _VarUInt64ByteSizeNoTag(message_size) + + # The size of the message. + total_size += message_size + return total_size + + +def TagByteSize(field_number): + """Returns the bytes required to serialize a tag with this field number.""" + # Just pass in type 0, since the type won't affect the tag+type size. + return _VarUInt64ByteSizeNoTag(PackTag(field_number, 0)) + + +# Private helper function for the *ByteSize() functions above. + +def _VarUInt64ByteSizeNoTag(uint64): + """Returns the number of bytes required to serialize a single varint + using boundary value comparisons. (unrolled loop optimization -WPierce) + uint64 must be unsigned. + """ + if uint64 <= 0x7f: return 1 + if uint64 <= 0x3fff: return 2 + if uint64 <= 0x1fffff: return 3 + if uint64 <= 0xfffffff: return 4 + if uint64 <= 0x7ffffffff: return 5 + if uint64 <= 0x3ffffffffff: return 6 + if uint64 <= 0x1ffffffffffff: return 7 + if uint64 <= 0xffffffffffffff: return 8 + if uint64 <= 0x7fffffffffffffff: return 9 + if uint64 > UINT64_MAX: + raise message.EncodeError('Value out of range: %d' % uint64) + return 10 + + +NON_PACKABLE_TYPES = ( + descriptor.FieldDescriptor.TYPE_STRING, + descriptor.FieldDescriptor.TYPE_GROUP, + descriptor.FieldDescriptor.TYPE_MESSAGE, + descriptor.FieldDescriptor.TYPE_BYTES +) + + +def IsTypePackable(field_type): + """Return true iff packable = true is valid for fields of this type. + + Args: + field_type: a FieldDescriptor::Type value. + + Returns: + True iff fields of this type are packable. + """ + return field_type not in NON_PACKABLE_TYPES diff --git a/lib/python3.12/site-packages/google/protobuf/json_format.py b/lib/python3.12/site-packages/google/protobuf/json_format.py new file mode 100644 index 0000000000000000000000000000000000000000..b671d3cc7c040acf4903174bb99c5824d23f6a4f --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/json_format.py @@ -0,0 +1,1114 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains routines for printing protocol messages in JSON format. + +Simple usage example: + + # Create a proto object and serialize it to a json format string. + message = my_proto_pb2.MyMessage(foo='bar') + json_string = json_format.MessageToJson(message) + + # Parse a json format string to proto object. + message = json_format.Parse(json_string, my_proto_pb2.MyMessage()) +""" + +__author__ = 'jieluo@google.com (Jie Luo)' + + +import base64 +from collections import OrderedDict +import json +import math +from operator import methodcaller +import re +import warnings + +from google.protobuf import descriptor +from google.protobuf import message_factory +from google.protobuf import symbol_database +from google.protobuf.internal import type_checkers + + +_INT_TYPES = frozenset([ + descriptor.FieldDescriptor.CPPTYPE_INT32, + descriptor.FieldDescriptor.CPPTYPE_UINT32, + descriptor.FieldDescriptor.CPPTYPE_INT64, + descriptor.FieldDescriptor.CPPTYPE_UINT64, +]) +_INT64_TYPES = frozenset([ + descriptor.FieldDescriptor.CPPTYPE_INT64, + descriptor.FieldDescriptor.CPPTYPE_UINT64, +]) +_FLOAT_TYPES = frozenset([ + descriptor.FieldDescriptor.CPPTYPE_FLOAT, + descriptor.FieldDescriptor.CPPTYPE_DOUBLE, +]) +_INFINITY = 'Infinity' +_NEG_INFINITY = '-Infinity' +_NAN = 'NaN' + +_UNPAIRED_SURROGATE_PATTERN = re.compile( + '[\ud800-\udbff](?![\udc00-\udfff])|(? self.max_recursion_depth: + raise ParseError( + 'Message too deep. Max recursion depth is {0}'.format( + self.max_recursion_depth + ) + ) + message_descriptor = message.DESCRIPTOR + full_name = message_descriptor.full_name + if not path: + path = message_descriptor.name + if _IsWrapperMessage(message_descriptor): + self._ConvertWrapperMessage(value, message, path) + elif full_name in _WKTJSONMETHODS: + methodcaller(_WKTJSONMETHODS[full_name][1], value, message, path)(self) + else: + self._ConvertFieldValuePair(value, message, path) + self.recursion_depth -= 1 + + def _ConvertFieldValuePair(self, js, message, path): + """Convert field value pairs into regular message. + + Args: + js: A JSON object to convert the field value pairs. + message: A regular protocol message to record the data. + path: parent path to log parse error info. + + Raises: + ParseError: In case of problems converting. + """ + names = [] + message_descriptor = message.DESCRIPTOR + fields_by_json_name = dict( + (f.json_name, f) for f in message_descriptor.fields + ) + + def _ClearFieldOrExtension(message, field): + if field.is_extension: + message.ClearExtension(field) + else: + message.ClearField(field.name) + + def _GetFieldOrExtension(message, field): + if field.is_extension: + return message.Extensions[field] + else: + return getattr(message, field.name) + + def _SetFieldOrExtension(message, field, value): + if field.is_extension: + message.Extensions[field] = value + else: + setattr(message, field.name, value) + + for name in js: + try: + field = fields_by_json_name.get(name, None) + if not field: + field = message_descriptor.fields_by_name.get(name, None) + if not field and _VALID_EXTENSION_NAME.match(name): + if not message_descriptor.is_extendable: + raise ParseError( + 'Message type {0} does not have extensions at {1}'.format( + message_descriptor.full_name, path + ) + ) + identifier = name[1:-1] # strip [] brackets + # pylint: disable=protected-access + field = message.Extensions._FindExtensionByName(identifier) + # pylint: enable=protected-access + if not field: + # Try looking for extension by the message type name, dropping the + # field name following the final . separator in full_name. + identifier = '.'.join(identifier.split('.')[:-1]) + # pylint: disable=protected-access + field = message.Extensions._FindExtensionByName(identifier) + # pylint: enable=protected-access + if not field: + if self.ignore_unknown_fields: + continue + raise ParseError( + ( + 'Message type "{0}" has no field named "{1}" at "{2}".\n' + ' Available Fields(except extensions): "{3}"' + ).format( + message_descriptor.full_name, + name, + path, + [f.json_name for f in message_descriptor.fields], + ) + ) + if name in names: + raise ParseError( + 'Message type "{0}" should not have multiple ' + '"{1}" fields at "{2}".'.format( + message.DESCRIPTOR.full_name, name, path + ) + ) + names.append(name) + value = js[name] + # Check no other oneof field is parsed. + if field.containing_oneof is not None and value is not None: + oneof_name = field.containing_oneof.name + if oneof_name in names: + raise ParseError( + 'Message type "{0}" should not have multiple ' + '"{1}" oneof fields at "{2}".'.format( + message.DESCRIPTOR.full_name, oneof_name, path + ) + ) + names.append(oneof_name) + + if value is None: + if ( + field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE + and field.message_type.full_name == 'google.protobuf.Value' + ): + sub_message = _GetFieldOrExtension(message, field) + sub_message.null_value = 0 + elif ( + field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM + and field.enum_type.full_name == 'google.protobuf.NullValue' + ): + _SetFieldOrExtension(message, field, 0) + else: + _ClearFieldOrExtension(message, field) + continue + + # Parse field value. + if _IsMapEntry(field): + _ClearFieldOrExtension(message, field) + self._ConvertMapFieldValue( + value, message, field, '{0}.{1}'.format(path, name) + ) + elif field.is_repeated: + _ClearFieldOrExtension(message, field) + if not isinstance(value, _LIST_LIKE): + raise ParseError( + 'repeated field {0} must be in [] which is {1} at {2}'.format( + name, value, path + ) + ) + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + # Repeated message field. + for index, item in enumerate(value): + sub_message = _GetFieldOrExtension(message, field).add() + # None is a null_value in Value. + if ( + item is None + and sub_message.DESCRIPTOR.full_name + != 'google.protobuf.Value' + ): + raise ParseError( + 'null is not allowed to be used as an element' + ' in a repeated field at {0}.{1}[{2}]'.format( + path, name, index + ) + ) + self.ConvertMessage( + item, sub_message, '{0}.{1}[{2}]'.format(path, name, index) + ) + else: + # Repeated scalar field. + for index, item in enumerate(value): + if item is None: + raise ParseError( + 'null is not allowed to be used as an element' + ' in a repeated field at {0}.{1}[{2}]'.format( + path, name, index + ) + ) + self._ConvertAndAppendScalar( + message, field, item, '{0}.{1}[{2}]'.format(path, name, index) + ) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + sub_message = _GetFieldOrExtension(message, field) + sub_message.SetInParent() + self.ConvertMessage(value, sub_message, '{0}.{1}'.format(path, name)) + else: + self._ConvertAndSetScalar( + message, field, value, '{0}.{1}'.format(path, name) + ) + except ParseError as e: + if field and field.containing_oneof is None: + raise ParseError( + 'Failed to parse {0} field: {1}.'.format(name, e) + ) from e + else: + raise ParseError(str(e)) from e + except ValueError as e: + raise ParseError( + 'Failed to parse {0} field: {1}.'.format(name, e) + ) from e + except TypeError as e: + raise ParseError( + 'Failed to parse {0} field: {1}.'.format(name, e) + ) from e + + def _ConvertAnyMessage(self, value, message, path): + """Convert a JSON representation into Any message.""" + if isinstance(value, dict) and not value: + return + try: + type_url = value['@type'] + except KeyError as e: + raise ParseError( + '@type is missing when parsing any message at {0}'.format(path) + ) from e + + try: + sub_message = _CreateMessageFromTypeUrl(type_url, self.descriptor_pool) + except TypeError as e: + raise ParseError('{0} at {1}'.format(e, path)) from e + message_descriptor = sub_message.DESCRIPTOR + full_name = message_descriptor.full_name + if _IsWrapperMessage(message_descriptor): + self._ConvertWrapperMessage( + value['value'], sub_message, '{0}.value'.format(path) + ) + elif full_name in _WKTJSONMETHODS: + # For well-known types (including nested Any), use ConvertMessage + # to ensure recursion depth is properly tracked + self.ConvertMessage( + value['value'], sub_message, '{0}.value'.format(path) + ) + else: + del value['@type'] + try: + self._ConvertFieldValuePair(value, sub_message, path) + finally: + value['@type'] = type_url + # Sets Any message + message.value = sub_message.SerializeToString() + message.type_url = type_url + + def _ConvertGenericMessage(self, value, message, path): + """Convert a JSON representation into message with FromJsonString.""" + # Duration, Timestamp, FieldMask have a FromJsonString method to do the + # conversion. Users can also call the method directly. + try: + message.FromJsonString(value) + except ValueError as e: + raise ParseError('{0} at {1}'.format(e, path)) from e + + def _ConvertValueMessage(self, value, message, path): + """Convert a JSON representation into Value message.""" + if isinstance(value, dict): + self._ConvertStructMessage(value, message.struct_value, path) + elif isinstance(value, _LIST_LIKE): + self._ConvertListOrTupleValueMessage(value, message.list_value, path) + elif value is None: + message.null_value = 0 + elif isinstance(value, bool): + message.bool_value = value + elif isinstance(value, str): + message.string_value = value + elif isinstance(value, _INT_OR_FLOAT): + message.number_value = value + else: + raise ParseError( + 'Value {0} has unexpected type {1} at {2}'.format( + value, type(value), path + ) + ) + + def _ConvertListOrTupleValueMessage(self, value, message, path): + """Convert a JSON representation into ListValue message.""" + if not isinstance(value, _LIST_LIKE): + raise ParseError( + 'ListValue must be in [] which is {0} at {1}'.format(value, path) + ) + message.ClearField('values') + for index, item in enumerate(value): + self._ConvertValueMessage( + item, message.values.add(), '{0}[{1}]'.format(path, index) + ) + + def _ConvertStructMessage(self, value, message, path): + """Convert a JSON representation into Struct message.""" + if not isinstance(value, dict): + raise ParseError( + 'Struct must be in a dict which is {0} at {1}'.format(value, path) + ) + # Clear will mark the struct as modified so it will be created even if + # there are no values. + message.Clear() + for key in value: + self._ConvertValueMessage( + value[key], message.fields[key], '{0}.{1}'.format(path, key) + ) + return + + def _ConvertWrapperMessage(self, value, message, path): + """Convert a JSON representation into Wrapper message.""" + field = message.DESCRIPTOR.fields_by_name['value'] + self._ConvertAndSetScalar( + message, field, value, path='{0}.value'.format(path) + ) + + def _ConvertMapFieldValue(self, value, message, field, path): + """Convert map field value for a message map field. + + Args: + value: A JSON object to convert the map field value. + message: A protocol message to record the converted data. + field: The descriptor of the map field to be converted. + path: parent path to log parse error info. + + Raises: + ParseError: In case of convert problems. + """ + if not isinstance(value, dict): + raise ParseError( + 'Map field {0} must be in a dict which is {1} at {2}'.format( + field.name, value, path + ) + ) + key_field = field.message_type.fields_by_name['key'] + value_field = field.message_type.fields_by_name['value'] + for key in value: + key_value = _ConvertScalarFieldValue( + key, key_field, '{0}.key'.format(path), True + ) + if value_field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + self.ConvertMessage( + value[key], + getattr(message, field.name)[key_value], + '{0}[{1}]'.format(path, key_value), + ) + else: + self._ConvertAndSetScalarToMapKey( + message, + field, + key_value, + value[key], + path='{0}[{1}]'.format(path, key_value), + ) + + def _ConvertAndSetScalar(self, message, field, js_value, path): + """Convert scalar from js_value and assign it to message.field.""" + try: + value = _ConvertScalarFieldValue(js_value, field, path) + if field.is_extension: + message.Extensions[field] = value + else: + setattr(message, field.name, value) + except EnumStringValueParseError: + if not self.ignore_unknown_fields: + raise + + def _ConvertAndAppendScalar(self, message, repeated_field, js_value, path): + """Convert scalar from js_value and append it to message.repeated_field.""" + try: + if repeated_field.is_extension: + repeated = message.Extensions[repeated_field] + else: + repeated = getattr(message, repeated_field.name) + value = _ConvertScalarFieldValue(js_value, repeated_field, path) + repeated.append(value) + except EnumStringValueParseError: + if not self.ignore_unknown_fields: + raise + + def _ConvertAndSetScalarToMapKey( + self, message, map_field, converted_key, js_value, path + ): + """Convert scalar from 'js_value' and add it to message.map_field[converted_key].""" + try: + getattr(message, map_field.name)[converted_key] = ( + _ConvertScalarFieldValue( + js_value, + map_field.message_type.fields_by_name['value'], + path, + ) + ) + except EnumStringValueParseError: + if not self.ignore_unknown_fields: + raise + + +def _ConvertScalarFieldValue(value, field, path, require_str=False): + """Convert a single scalar field value. + + Args: + value: A scalar value to convert the scalar field value. + field: The descriptor of the field to convert. + path: parent path to log parse error info. + require_str: If True, the field value must be a str. + + Returns: + The converted scalar field value + + Raises: + ParseError: In case of convert problems. + EnumStringValueParseError: In case of unknown enum string value. + """ + try: + if field.cpp_type in _INT_TYPES: + return _ConvertInteger(value) + elif field.cpp_type in _FLOAT_TYPES: + return _ConvertFloat(value, field) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: + return _ConvertBool(value, require_str) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: + if field.type == descriptor.FieldDescriptor.TYPE_BYTES: + if isinstance(value, str): + encoded = value.encode('utf-8') + else: + encoded = value + # Add extra padding '=' + padded_value = encoded + b'=' * (4 - len(encoded) % 4) + return base64.urlsafe_b64decode(padded_value) + else: + # Checking for unpaired surrogates appears to be unreliable, + # depending on the specific Python version, so we check manually. + if _UNPAIRED_SURROGATE_PATTERN.search(value): + raise ParseError('Unpaired surrogate') + return value + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: + # Convert an enum value. + enum_value = field.enum_type.values_by_name.get(value, None) + if enum_value is None: + try: + number = int(value) + enum_value = field.enum_type.values_by_number.get(number, None) + except ValueError as e: + # Since parsing to integer failed and lookup in values_by_name didn't + # find this name, we have an enum string value which is unknown. + raise EnumStringValueParseError( + 'Invalid enum value {0} for enum type {1}'.format( + value, field.enum_type.full_name + ) + ) from e + if enum_value is None: + if field.enum_type.is_closed: + raise ParseError( + 'Invalid enum value {0} for enum type {1}'.format( + value, field.enum_type.full_name + ) + ) + else: + return number + return enum_value.number + except EnumStringValueParseError as e: + raise EnumStringValueParseError('{0} at {1}'.format(e, path)) from e + except ParseError as e: + raise ParseError('{0} at {1}'.format(e, path)) from e + + +def _ConvertInteger(value): + """Convert an integer. + + Args: + value: A scalar value to convert. + + Returns: + The integer value. + + Raises: + ParseError: If an integer couldn't be consumed. + """ + if isinstance(value, float) and not value.is_integer(): + raise ParseError("Couldn't parse integer: {0}".format(value)) + + if isinstance(value, str) and value.find(' ') != -1: + raise ParseError('Couldn\'t parse integer: "{0}"'.format(value)) + + if isinstance(value, bool): + raise ParseError( + 'Bool value {0} is not acceptable for integer field'.format(value) + ) + + try: + return int(value) + except ValueError as e: + # Attempt to parse as an integer-valued float. + try: + f = float(value) + except ValueError: + # Raise the original exception for the int parse. + raise e # pylint: disable=raise-missing-from + if not f.is_integer(): + raise ParseError( + 'Couldn\'t parse non-integer string: "{0}"'.format(value) + ) from e + return int(f) + + +def _ConvertFloat(value, field): + """Convert an floating point number.""" + if isinstance(value, float): + if math.isnan(value): + raise ParseError('Couldn\'t parse NaN, use quoted "NaN" instead') + if math.isinf(value): + if value > 0: + raise ParseError( + "Couldn't parse Infinity or value too large, " + 'use quoted "Infinity" instead' + ) + else: + raise ParseError( + "Couldn't parse -Infinity or value too small, " + 'use quoted "-Infinity" instead' + ) + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT: + # pylint: disable=protected-access + if value > type_checkers._FLOAT_MAX: + raise ParseError('Float value too large') + # pylint: disable=protected-access + if value < type_checkers._FLOAT_MIN: + raise ParseError('Float value too small') + if value == 'nan': + raise ParseError('Couldn\'t parse float "nan", use "NaN" instead') + try: + # Assume Python compatible syntax. + return float(value) + except ValueError as e: + # Check alternative spellings. + if value == _NEG_INFINITY: + return float('-inf') + elif value == _INFINITY: + return float('inf') + elif value == _NAN: + return float('nan') + else: + raise ParseError("Couldn't parse float: {0}".format(value)) from e + + +def _ConvertBool(value, require_str): + """Convert a boolean value. + + Args: + value: A scalar value to convert. + require_str: If True, value must be a str. + + Returns: + The bool parsed. + + Raises: + ParseError: If a boolean value couldn't be consumed. + """ + if require_str: + if value == 'true': + return True + elif value == 'false': + return False + else: + raise ParseError('Expected "true" or "false", not {0}'.format(value)) + + if not isinstance(value, bool): + raise ParseError('Expected true or false without quotes') + return value + + +_WKTJSONMETHODS = { + 'google.protobuf.Any': ['_AnyMessageToJsonObject', '_ConvertAnyMessage'], + 'google.protobuf.Duration': [ + '_GenericMessageToJsonObject', + '_ConvertGenericMessage', + ], + 'google.protobuf.FieldMask': [ + '_GenericMessageToJsonObject', + '_ConvertGenericMessage', + ], + 'google.protobuf.ListValue': [ + '_ListValueMessageToJsonObject', + '_ConvertListOrTupleValueMessage', + ], + 'google.protobuf.Struct': [ + '_StructMessageToJsonObject', + '_ConvertStructMessage', + ], + 'google.protobuf.Timestamp': [ + '_GenericMessageToJsonObject', + '_ConvertGenericMessage', + ], + 'google.protobuf.Value': [ + '_ValueMessageToJsonObject', + '_ConvertValueMessage', + ], +} diff --git a/lib/python3.12/site-packages/google/protobuf/message.py b/lib/python3.12/site-packages/google/protobuf/message.py new file mode 100644 index 0000000000000000000000000000000000000000..d48f4f409d055f6e47fa7985a8be0c96bdb57797 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/message.py @@ -0,0 +1,448 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# TODO: We should just make these methods all "pure-virtual" and move +# all implementation out, into reflection.py for now. + + +"""Contains an abstract base class for protocol messages.""" + +__author__ = 'robinson@google.com (Will Robinson)' + +_INCONSISTENT_MESSAGE_ATTRIBUTES = ('Extensions',) + + +class Error(Exception): + """Base error type for this module.""" + pass + + +class DecodeError(Error): + """Exception raised when deserializing messages.""" + pass + + +class EncodeError(Error): + """Exception raised when serializing messages.""" + pass + + +class Message(object): + + """Abstract base class for protocol messages. + + Protocol message classes are almost always generated by the protocol + compiler. These generated types subclass Message and implement the methods + shown below. + """ + + # TODO: Link to an HTML document here. + + # TODO: Document that instances of this class will also + # have an Extensions attribute with __getitem__ and __setitem__. + # Again, not sure how to best convey this. + + # TODO: Document these fields and methods. + + __slots__ = [] + + #: The :class:`google.protobuf.Descriptor` + # for this message type. + DESCRIPTOR = None + + def __deepcopy__(self, memo=None): + clone = type(self)() + clone.MergeFrom(self) + return clone + + def __dir__(self): + """Provides the list of all accessible Message attributes.""" + message_attributes = set(super().__dir__()) + + # TODO: Remove this once the UPB implementation is improved. + # The UPB proto implementation currently doesn't provide proto fields as + # attributes and they have to added. + if self.DESCRIPTOR is not None: + for field in self.DESCRIPTOR.fields: + message_attributes.add(field.name) + + # The Fast C++ proto implementation provides inaccessible attributes that + # have to be removed. + for attribute in _INCONSISTENT_MESSAGE_ATTRIBUTES: + if attribute not in message_attributes: + continue + try: + getattr(self, attribute) + except AttributeError: + message_attributes.remove(attribute) + + return sorted(message_attributes) + + def __eq__(self, other_msg): + """Recursively compares two messages by value and structure.""" + raise NotImplementedError + + def __ne__(self, other_msg): + # Can't just say self != other_msg, since that would infinitely recurse. :) + return not self == other_msg + + def __hash__(self): + raise TypeError('unhashable object') + + def __str__(self): + """Outputs a human-readable representation of the message.""" + raise NotImplementedError + + def __unicode__(self): + """Outputs a human-readable representation of the message.""" + raise NotImplementedError + + def __contains__(self, field_name_or_key): + """Checks if a certain field is set for the message. + + Has presence fields return true if the field is set, false if the field is + not set. Fields without presence do raise `ValueError` (this includes + repeated fields, map fields, and implicit presence fields). + + If field_name is not defined in the message descriptor, `ValueError` will + be raised. + Note: WKT Struct checks if the key is contained in fields. ListValue checks + if the item is contained in the list. + + Args: + field_name_or_key: For Struct, the key (str) of the fields map. For + ListValue, any type that may be contained in the list. For other + messages, name of the field (str) to check for presence. + + Returns: + bool: For Struct, whether the item is contained in fields. For ListValue, + whether the item is contained in the list. For other message, + whether a value has been set for the named field. + + Raises: + ValueError: For normal messages, if the `field_name_or_key` is not a + member of this message or `field_name_or_key` is not a string. + """ + raise NotImplementedError + + def MergeFrom(self, other_msg): + """Merges the contents of the specified message into current message. + + This method merges the contents of the specified message into the current + message. Singular fields that are set in the specified message overwrite + the corresponding fields in the current message. Repeated fields are + appended. Singular sub-messages and groups are recursively merged. + + Args: + other_msg (Message): A message to merge into the current message. + """ + raise NotImplementedError + + def CopyFrom(self, other_msg): + """Copies the content of the specified message into the current message. + + The method clears the current message and then merges the specified + message using MergeFrom. + + Args: + other_msg (Message): A message to copy into the current one. + """ + if self is other_msg: + return + self.Clear() + self.MergeFrom(other_msg) + + def Clear(self): + """Clears all data that was set in the message.""" + raise NotImplementedError + + def SetInParent(self): + """Mark this as present in the parent. + + This normally happens automatically when you assign a field of a + sub-message, but sometimes you want to make the sub-message + present while keeping it empty. If you find yourself using this, + you may want to reconsider your design. + """ + raise NotImplementedError + + def IsInitialized(self): + """Checks if the message is initialized. + + Returns: + bool: The method returns True if the message is initialized (i.e. all of + its required fields are set). + """ + raise NotImplementedError + + # TODO: MergeFromString() should probably return None and be + # implemented in terms of a helper that returns the # of bytes read. Our + # deserialization routines would use the helper when recursively + # deserializing, but the end user would almost always just want the no-return + # MergeFromString(). + + def MergeFromString(self, serialized): + """Merges serialized protocol buffer data into this message. + + When we find a field in `serialized` that is already present + in this message: + + - If it's a "repeated" field, we append to the end of our list. + - Else, if it's a scalar, we overwrite our field. + - Else, (it's a nonrepeated composite), we recursively merge + into the existing composite. + + Args: + serialized (bytes): Any object that allows us to call + ``memoryview(serialized)`` to access a string of bytes using the + buffer interface. + + Returns: + int: The number of bytes read from `serialized`. + For non-group messages, this will always be `len(serialized)`, + but for messages which are actually groups, this will + generally be less than `len(serialized)`, since we must + stop when we reach an ``END_GROUP`` tag. Note that if + we *do* stop because of an ``END_GROUP`` tag, the number + of bytes returned does not include the bytes + for the ``END_GROUP`` tag information. + + Raises: + DecodeError: if the input cannot be parsed. + """ + # TODO: Document handling of unknown fields. + # TODO: When we switch to a helper, this will return None. + raise NotImplementedError + + def ParseFromString(self, serialized): + """Parse serialized protocol buffer data in binary form into this message. + + Like :func:`MergeFromString()`, except we clear the object first. + + Raises: + message.DecodeError if the input cannot be parsed. + """ + self.Clear() + return self.MergeFromString(serialized) + + def SerializeToString(self, **kwargs): + """Serializes the protocol message to a binary string. + + Keyword Args: + deterministic (bool): If true, requests deterministic serialization + of the protobuf, with predictable ordering of map keys. + + Returns: + A binary string representation of the message if all of the required + fields in the message are set (i.e. the message is initialized). + + Raises: + EncodeError: if the message isn't initialized (see :func:`IsInitialized`). + """ + raise NotImplementedError + + def SerializePartialToString(self, **kwargs): + """Serializes the protocol message to a binary string. + + This method is similar to SerializeToString but doesn't check if the + message is initialized. + + Keyword Args: + deterministic (bool): If true, requests deterministic serialization + of the protobuf, with predictable ordering of map keys. + + Returns: + bytes: A serialized representation of the partial message. + """ + raise NotImplementedError + + # TODO: Decide whether we like these better + # than auto-generated has_foo() and clear_foo() methods + # on the instances themselves. This way is less consistent + # with C++, but it makes reflection-type access easier and + # reduces the number of magically autogenerated things. + # + # TODO: Be sure to document (and test) exactly + # which field names are accepted here. Are we case-sensitive? + # What do we do with fields that share names with Python keywords + # like 'lambda' and 'yield'? + # + # nnorwitz says: + # """ + # Typically (in python), an underscore is appended to names that are + # keywords. So they would become lambda_ or yield_. + # """ + def ListFields(self): + """Returns a list of (FieldDescriptor, value) tuples for present fields. + + A message field is non-empty if HasField() would return true. A singular + primitive field is non-empty if HasField() would return true in proto2 or it + is non zero in proto3. A repeated field is non-empty if it contains at least + one element. The fields are ordered by field number. + + Returns: + list[tuple(FieldDescriptor, value)]: field descriptors and values + for all fields in the message which are not empty. The values vary by + field type. + """ + raise NotImplementedError + + def HasField(self, field_name): + """Checks if a certain field is set for the message. + + For a oneof group, checks if any field inside is set. Note that if the + field_name is not defined in the message descriptor, :exc:`ValueError` will + be raised. + + Args: + field_name (str): The name of the field to check for presence. + + Returns: + bool: Whether a value has been set for the named field. + + Raises: + ValueError: if the `field_name` is not a member of this message. + """ + raise NotImplementedError + + def ClearField(self, field_name): + """Clears the contents of a given field. + + Inside a oneof group, clears the field set. If the name neither refers to a + defined field or oneof group, :exc:`ValueError` is raised. + + Args: + field_name (str): The name of the field to check for presence. + + Raises: + ValueError: if the `field_name` is not a member of this message. + """ + raise NotImplementedError + + def WhichOneof(self, oneof_group): + """Returns the name of the field that is set inside a oneof group. + + If no field is set, returns None. + + Args: + oneof_group (str): the name of the oneof group to check. + + Returns: + str or None: The name of the group that is set, or None. + + Raises: + ValueError: no group with the given name exists + """ + raise NotImplementedError + + def HasExtension(self, field_descriptor): + """Checks if a certain extension is present for this message. + + Extensions are retrieved using the :attr:`Extensions` mapping (if present). + + Args: + field_descriptor: The field descriptor for the extension to check. + + Returns: + bool: Whether the extension is present for this message. + + Raises: + KeyError: if the extension is repeated. Similar to repeated fields, + there is no separate notion of presence: a "not present" repeated + extension is an empty list. + """ + raise NotImplementedError + + def ClearExtension(self, field_descriptor): + """Clears the contents of a given extension. + + Args: + field_descriptor: The field descriptor for the extension to clear. + """ + raise NotImplementedError + + def UnknownFields(self): + """Returns the UnknownFieldSet. + + Returns: + UnknownFieldSet: The unknown fields stored in this message. + """ + raise NotImplementedError + + def DiscardUnknownFields(self): + """Clears all fields in the :class:`UnknownFieldSet`. + + This operation is recursive for nested message. + """ + raise NotImplementedError + + def ByteSize(self): + """Returns the serialized size of this message. + + Recursively calls ByteSize() on all contained messages. + + Returns: + int: The number of bytes required to serialize this message. + """ + raise NotImplementedError + + @classmethod + def FromString(cls, s): + raise NotImplementedError + + def _SetListener(self, message_listener): + """Internal method used by the protocol message implementation. + Clients should not call this directly. + + Sets a listener that this message will call on certain state transitions. + + The purpose of this method is to register back-edges from children to + parents at runtime, for the purpose of setting "has" bits and + byte-size-dirty bits in the parent and ancestor objects whenever a child or + descendant object is modified. + + If the client wants to disconnect this Message from the object tree, she + explicitly sets callback to None. + + If message_listener is None, unregisters any existing listener. Otherwise, + message_listener must implement the MessageListener interface in + internal/message_listener.py, and we discard any listener registered + via a previous _SetListener() call. + """ + raise NotImplementedError + + def __getstate__(self): + """Support the pickle protocol.""" + return dict(serialized=self.SerializePartialToString()) + + def __setstate__(self, state): + """Support the pickle protocol.""" + self.__init__() + serialized = state['serialized'] + # On Python 3, using encoding='latin1' is required for unpickling + # protos pickled by Python 2. + if not isinstance(serialized, bytes): + serialized = serialized.encode('latin1') + self.ParseFromString(serialized) + + def __reduce__(self): + message_descriptor = self.DESCRIPTOR + if message_descriptor.containing_type is None: + return type(self), (), self.__getstate__() + # the message type must be nested. + # Python does not pickle nested classes; use the symbol_database on the + # receiving end. + container = message_descriptor + return (_InternalConstructMessage, (container.full_name,), + self.__getstate__()) + + +def _InternalConstructMessage(full_name): + """Constructs a nested message.""" + from google.protobuf import symbol_database # pylint:disable=g-import-not-at-top + + return symbol_database.Default().GetSymbol(full_name)() diff --git a/lib/python3.12/site-packages/google/protobuf/message_factory.py b/lib/python3.12/site-packages/google/protobuf/message_factory.py new file mode 100644 index 0000000000000000000000000000000000000000..3bc4d92867c08199a582d515e6e326e3bc29ce82 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/message_factory.py @@ -0,0 +1,190 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Provides a factory class for generating dynamic messages. + +The easiest way to use this class is if you have access to the FileDescriptor +protos containing the messages you want to create you can just do the following: + +message_classes = message_factory.GetMessages(iterable_of_file_descriptors) +my_proto_instance = message_classes['some.proto.package.MessageName']() +""" + +__author__ = 'matthewtoia@google.com (Matt Toia)' + +import warnings + +from google.protobuf import descriptor_pool +from google.protobuf import message +from google.protobuf.internal import api_implementation + +if api_implementation.Type() == 'python': + from google.protobuf.internal import python_message as message_impl +else: + from google.protobuf.pyext import cpp_message as message_impl # pylint: disable=g-import-not-at-top + + +# The type of all Message classes. +_GENERATED_PROTOCOL_MESSAGE_TYPE = message_impl.GeneratedProtocolMessageType + + +def GetMessageClass(descriptor): + """Obtains a proto2 message class based on the passed in descriptor. + + Passing a descriptor with a fully qualified name matching a previous + invocation will cause the same class to be returned. + + Args: + descriptor: The descriptor to build from. + + Returns: + A class describing the passed in descriptor. + """ + concrete_class = getattr(descriptor, '_concrete_class', None) + if concrete_class: + return concrete_class + return _InternalCreateMessageClass(descriptor) + + +def GetMessageClassesForFiles(files, pool): + """Gets all the messages from specified files. + + This will find and resolve dependencies, failing if the descriptor + pool cannot satisfy them. + + This will not return the classes for nested types within those classes, for + those, use GetMessageClass() on the nested types within their containing + messages. + + For example, for the message: + + message NestedTypeMessage { + message NestedType { + string data = 1; + } + NestedType nested = 1; + } + + NestedTypeMessage will be in the result, but not + NestedTypeMessage.NestedType. + + Args: + files: The file names to extract messages from. + pool: The descriptor pool to find the files including the dependent files. + + Returns: + A dictionary mapping proto names to the message classes. + """ + result = {} + for file_name in files: + file_desc = pool.FindFileByName(file_name) + for desc in file_desc.message_types_by_name.values(): + result[desc.full_name] = GetMessageClass(desc) + + # While the extension FieldDescriptors are created by the descriptor pool, + # the python classes created in the factory need them to be registered + # explicitly, which is done below. + # + # The call to RegisterExtension will specifically check if the + # extension was already registered on the object and either + # ignore the registration if the original was the same, or raise + # an error if they were different. + + for extension in file_desc.extensions_by_name.values(): + _ = GetMessageClass(extension.containing_type) + if api_implementation.Type() != 'python': + # TODO: Remove this check here. Duplicate extension + # register check should be in descriptor_pool. + if extension is not pool.FindExtensionByNumber( + extension.containing_type, extension.number + ): + raise ValueError('Double registration of Extensions') + # Recursively load protos for extension field, in order to be able to + # fully represent the extension. This matches the behavior for regular + # fields too. + if extension.message_type: + GetMessageClass(extension.message_type) + return result + + +def _InternalCreateMessageClass(descriptor): + """Builds a proto2 message class based on the passed in descriptor. + + Args: + descriptor: The descriptor to build from. + + Returns: + A class describing the passed in descriptor. + """ + descriptor_name = descriptor.name + result_class = _GENERATED_PROTOCOL_MESSAGE_TYPE( + descriptor_name, + (message.Message,), + { + 'DESCRIPTOR': descriptor, + # If module not set, it wrongly points to message_factory module. + '__module__': None, + }, + ) + for field in descriptor.fields: + if field.message_type: + GetMessageClass(field.message_type) + + for extension in result_class.DESCRIPTOR.extensions: + extended_class = GetMessageClass(extension.containing_type) + if api_implementation.Type() != 'python': + # TODO: Remove this check here. Duplicate extension + # register check should be in descriptor_pool. + pool = extension.containing_type.file.pool + if extension is not pool.FindExtensionByNumber( + extension.containing_type, extension.number + ): + raise ValueError('Double registration of Extensions') + if extension.message_type: + GetMessageClass(extension.message_type) + return result_class + + +# Deprecated. Please use GetMessageClass() or GetMessageClassesForFiles() +# method above instead. +class MessageFactory(object): + """Factory for creating Proto2 messages from descriptors in a pool.""" + + def __init__(self, pool=None): + """Initializes a new factory.""" + self.pool = pool or descriptor_pool.DescriptorPool() + + +def GetMessages(file_protos, pool=None): + """Builds a dictionary of all the messages available in a set of files. + + Args: + file_protos: Iterable of FileDescriptorProto to build messages out of. + pool: The descriptor pool to add the file protos. + + Returns: + A dictionary mapping proto names to the message classes. This will include + any dependent messages as well as any messages defined in the same file as + a specified message. + """ + # The cpp implementation of the protocol buffer library requires to add the + # message in topological order of the dependency graph. + des_pool = pool or descriptor_pool.DescriptorPool() + file_by_name = {file_proto.name: file_proto for file_proto in file_protos} + + def _AddFile(file_proto): + for dependency in file_proto.dependency: + if dependency in file_by_name: + # Remove from elements to be visited, in order to cut cycles. + _AddFile(file_by_name.pop(dependency)) + des_pool.Add(file_proto) + + while file_by_name: + _AddFile(file_by_name.popitem()[1]) + return GetMessageClassesForFiles( + [file_proto.name for file_proto in file_protos], des_pool + ) diff --git a/lib/python3.12/site-packages/google/protobuf/proto.py b/lib/python3.12/site-packages/google/protobuf/proto.py new file mode 100644 index 0000000000000000000000000000000000000000..1e6291c3ae4a6908b66437648e6f550e1113acc0 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/proto.py @@ -0,0 +1,153 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains the Nextgen Pythonic protobuf APIs.""" + +import io +from typing import Text, Type, TypeVar + +from google.protobuf.internal import decoder +from google.protobuf.internal import encoder +from google.protobuf.message import Message + +_MESSAGE = TypeVar('_MESSAGE', bound='Message') + + +def serialize(message: _MESSAGE, deterministic: bool = None) -> bytes: + """Return the serialized proto. + + Args: + message: The proto message to be serialized. + deterministic: If true, requests deterministic serialization + of the protobuf, with predictable ordering of map keys. + + Returns: + A binary bytes representation of the message. + """ + return message.SerializeToString(deterministic=deterministic) + + +def parse(message_class: Type[_MESSAGE], payload: bytes) -> _MESSAGE: + """Given a serialized data in binary form, deserialize it into a Message. + + Args: + message_class: The message meta class. + payload: A serialized bytes in binary form. + + Returns: + A new message deserialized from payload. + """ + new_message = message_class() + new_message.ParseFromString(payload) + return new_message + + +def serialize_length_prefixed(message: _MESSAGE, output: io.BytesIO) -> None: + """Writes the size of the message as a varint and the serialized message. + + Writes the size of the message as a varint and then the serialized message. + This allows more data to be written to the output after the message. Use + parse_length_prefixed to parse messages written by this method. + + The output stream must be buffered, e.g. using + https://docs.python.org/3/library/io.html#buffered-streams. + + Example usage: + out = io.BytesIO() + for msg in message_list: + proto.serialize_length_prefixed(msg, out) + + Args: + message: The protocol buffer message that should be serialized. + output: BytesIO or custom buffered IO that data should be written to. + """ + size = message.ByteSize() + encoder._VarintEncoder()(output.write, size) + out_size = output.write(serialize(message)) + + if out_size != size: + raise TypeError( + 'Failed to write complete message (wrote: %d, expected: %d)' + '. Ensure output is using buffered IO.' % (out_size, size) + ) + + +def parse_length_prefixed( + message_class: Type[_MESSAGE], input_bytes: io.BytesIO +) -> _MESSAGE: + """Parse a message from input_bytes. + + Args: + message_class: The protocol buffer message class that parser should parse. + input_bytes: A buffered input. + + Example usage: + while True: + msg = proto.parse_length_prefixed(message_class, input_bytes) + if msg is None: + break + ... + + Returns: + A parsed message if successful. None if input_bytes is at EOF. + """ + size = decoder._DecodeVarint(input_bytes) + if size is None: + # It is the end of buffered input. See example usage in the + # API description. + return None + + message = message_class() + + if size == 0: + return message + + parsed_size = message.ParseFromString(input_bytes.read(size)) + if parsed_size != size: + raise ValueError( + 'Truncated message or non-buffered input_bytes: ' + 'Expected {0} bytes but only {1} bytes parsed for ' + '{2}.'.format(size, parsed_size, message.DESCRIPTOR.name) + ) + return message + + +def byte_size(message: Message) -> int: + """Returns the serialized size of this message. + + Args: + message: A proto message. + + Returns: + int: The number of bytes required to serialize this message. + """ + return message.ByteSize() + + +def clear_message(message: Message) -> None: + """Clears all data that was set in the message. + + Args: + message: The proto message to be cleared. + """ + message.Clear() + + +def clear_field(message: Message, field_name: Text) -> None: + """Clears the contents of a given field. + + Inside a oneof group, clears the field set. If the name neither refers to a + defined field or oneof group, :exc:`ValueError` is raised. + + Args: + message: The proto message. + field_name (str): The name of the field to be cleared. + + Raises: + ValueError: if the `field_name` is not a member of this message. + """ + message.ClearField(field_name) diff --git a/lib/python3.12/site-packages/google/protobuf/proto_builder.py b/lib/python3.12/site-packages/google/protobuf/proto_builder.py new file mode 100644 index 0000000000000000000000000000000000000000..803d0040a99b6347d12c1e3891e0928ec8a4751f --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/proto_builder.py @@ -0,0 +1,111 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Dynamic Protobuf class creator.""" + +from collections import OrderedDict +import hashlib +import os + +from google.protobuf import descriptor_pb2 +from google.protobuf import descriptor +from google.protobuf import descriptor_pool +from google.protobuf import message_factory + + +def _GetMessageFromFactory(pool, full_name): + """Get a proto class from the MessageFactory by name. + + Args: + pool: a descriptor pool. + full_name: str, the fully qualified name of the proto type. + Returns: + A class, for the type identified by full_name. + Raises: + KeyError, if the proto is not found in the factory's descriptor pool. + """ + proto_descriptor = pool.FindMessageTypeByName(full_name) + proto_cls = message_factory.GetMessageClass(proto_descriptor) + return proto_cls + + +def MakeSimpleProtoClass(fields, full_name=None, pool=None): + """Create a Protobuf class whose fields are basic types. + + Note: this doesn't validate field names! + + Args: + fields: dict of {name: field_type} mappings for each field in the proto. If + this is an OrderedDict the order will be maintained, otherwise the + fields will be sorted by name. + full_name: optional str, the fully-qualified name of the proto type. + pool: optional DescriptorPool instance. + Returns: + a class, the new protobuf class with a FileDescriptor. + """ + pool_instance = pool or descriptor_pool.DescriptorPool() + if full_name is not None: + try: + proto_cls = _GetMessageFromFactory(pool_instance, full_name) + return proto_cls + except KeyError: + # The factory's DescriptorPool doesn't know about this class yet. + pass + + # Get a list of (name, field_type) tuples from the fields dict. If fields was + # an OrderedDict we keep the order, but otherwise we sort the field to ensure + # consistent ordering. + field_items = fields.items() + if not isinstance(fields, OrderedDict): + field_items = sorted(field_items) + + # Use a consistent file name that is unlikely to conflict with any imported + # proto files. + fields_hash = hashlib.sha1() + for f_name, f_type in field_items: + fields_hash.update(f_name.encode('utf-8')) + fields_hash.update(str(f_type).encode('utf-8')) + proto_file_name = fields_hash.hexdigest() + '.proto' + + # If the proto is anonymous, use the same hash to name it. + if full_name is None: + full_name = ('net.proto2.python.public.proto_builder.AnonymousProto_' + + fields_hash.hexdigest()) + try: + proto_cls = _GetMessageFromFactory(pool_instance, full_name) + return proto_cls + except KeyError: + # The factory's DescriptorPool doesn't know about this class yet. + pass + + # This is the first time we see this proto: add a new descriptor to the pool. + pool_instance.Add( + _MakeFileDescriptorProto(proto_file_name, full_name, field_items)) + return _GetMessageFromFactory(pool_instance, full_name) + + +def _MakeFileDescriptorProto(proto_file_name, full_name, field_items): + """Populate FileDescriptorProto for MessageFactory's DescriptorPool.""" + package, name = full_name.rsplit('.', 1) + file_proto = descriptor_pb2.FileDescriptorProto() + file_proto.name = os.path.join(package.replace('.', '/'), proto_file_name) + file_proto.package = package + desc_proto = file_proto.message_type.add() + desc_proto.name = name + for f_number, (f_name, f_type) in enumerate(field_items, 1): + field_proto = desc_proto.field.add() + field_proto.name = f_name + # # If the number falls in the reserved range, reassign it to the correct + # # number after the range. + if f_number >= descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER: + f_number += ( + descriptor.FieldDescriptor.LAST_RESERVED_FIELD_NUMBER - + descriptor.FieldDescriptor.FIRST_RESERVED_FIELD_NUMBER + 1) + field_proto.number = f_number + field_proto.label = descriptor_pb2.FieldDescriptorProto.LABEL_OPTIONAL + field_proto.type = f_type + return file_proto diff --git a/lib/python3.12/site-packages/google/protobuf/proto_json.py b/lib/python3.12/site-packages/google/protobuf/proto_json.py new file mode 100644 index 0000000000000000000000000000000000000000..ea670ab0810bfcb43abdc0b1ef71fbfdf95ecc68 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/proto_json.py @@ -0,0 +1,83 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains the Nextgen Pythonic Protobuf JSON APIs.""" + +from typing import Optional, Type + +from google.protobuf.message import Message +from google.protobuf.descriptor_pool import DescriptorPool +from google.protobuf import json_format + +def serialize( + message: Message, + always_print_fields_with_no_presence: bool=False, + preserving_proto_field_name: bool=False, + use_integers_for_enums: bool=False, + descriptor_pool: Optional[DescriptorPool]=None, + float_precision: int=None, +) -> dict: + """Converts protobuf message to a dictionary. + + When the dictionary is encoded to JSON, it conforms to proto3 JSON spec. + + Args: + message: The protocol buffers message instance to serialize. + always_print_fields_with_no_presence: If True, fields without + presence (implicit presence scalars, repeated fields, and map fields) will + always be serialized. Any field that supports presence is not affected by + this option (including singular message fields and oneof fields). + preserving_proto_field_name: If True, use the original proto field names as + defined in the .proto file. If False, convert the field names to + lowerCamelCase. + use_integers_for_enums: If true, print integers instead of enum names. + descriptor_pool: A Descriptor Pool for resolving types. If None use the + default. + float_precision: If set, use this to specify float field valid digits. + + Returns: + A dict representation of the protocol buffer message. + """ + return json_format.MessageToDict( + message, + always_print_fields_with_no_presence=always_print_fields_with_no_presence, + preserving_proto_field_name=preserving_proto_field_name, + use_integers_for_enums=use_integers_for_enums, + float_precision=float_precision, + ) + +def parse( + message_class: Type[Message], + js_dict: dict, + ignore_unknown_fields: bool=False, + descriptor_pool: Optional[DescriptorPool]=None, + max_recursion_depth: int=100 +) -> Message: + """Parses a JSON dictionary representation into a message. + + Args: + message_class: The message meta class. + js_dict: Dict representation of a JSON message. + ignore_unknown_fields: If True, do not raise errors for unknown fields. + descriptor_pool: A Descriptor Pool for resolving types. If None use the + default. + max_recursion_depth: max recursion depth of JSON message to be deserialized. + JSON messages over this depth will fail to be deserialized. Default value + is 100. + + Returns: + A new message passed from json_dict. + """ + new_message = message_class() + json_format.ParseDict( + js_dict=js_dict, + message=new_message, + ignore_unknown_fields=ignore_unknown_fields, + descriptor_pool=descriptor_pool, + max_recursion_depth=max_recursion_depth, + ) + return new_message diff --git a/lib/python3.12/site-packages/google/protobuf/proto_text.py b/lib/python3.12/site-packages/google/protobuf/proto_text.py new file mode 100644 index 0000000000000000000000000000000000000000..169fb4fce21f9ed8afdc90ec64abf0b784a3c60f --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/proto_text.py @@ -0,0 +1,129 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2025 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains the Nextgen Pythonic Protobuf Text Format APIs.""" +from typing import AnyStr, Callable, Optional, Text, Type, Union + +from google.protobuf import text_format +from google.protobuf.descriptor_pool import DescriptorPool +from google.protobuf.message import Message + +_MsgFormatter = Callable[[Message, Union[int, bool], bool], Optional[Text]] + + +def serialize( + message: Message, + as_utf8: bool = True, + as_one_line: bool = False, + use_short_repeated_primitives: bool = False, + pointy_brackets: bool = False, + use_index_order: bool = False, + float_format: Optional[str] = None, + double_format: Optional[str] = None, + use_field_number: bool = False, + descriptor_pool: Optional[DescriptorPool] = None, + indent: int = 0, + message_formatter: Optional[_MsgFormatter] = None, + print_unknown_fields: bool = False, + force_colon: bool = False, +) -> str: + """Convert protobuf message to text format. + + Double values can be formatted compactly with 15 digits of + precision (which is the most that IEEE 754 "double" can guarantee) + using double_format='.15g'. To ensure that converting to text and back to a + proto will result in an identical value, double_format='.17g' should be used. + + Args: + message: The protocol buffers message. + as_utf8: Return unescaped Unicode for non-ASCII characters. + as_one_line: Don't introduce newlines between fields. + use_short_repeated_primitives: Use short repeated format for primitives. + pointy_brackets: If True, use angle brackets instead of curly braces for + nesting. + use_index_order: If True, fields of a proto message will be printed using + the order defined in source code instead of the field number, extensions + will be printed at the end of the message and their relative order is + determined by the extension number. By default, use the field number + order. + float_format (str): If set, use this to specify float field formatting (per + the "Format Specification Mini-Language"); otherwise, shortest float that + has same value in wire will be printed. Also affect double field if + double_format is not set but float_format is set. + double_format (str): If set, use this to specify double field formatting + (per the "Format Specification Mini-Language"); if it is not set but + float_format is set, use float_format. Otherwise, use ``str()`` + use_field_number: If True, print field numbers instead of names. + descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types. + indent (int): The initial indent level, in terms of spaces, for pretty + print. + message_formatter (function(message, indent, as_one_line) -> unicode|None): + Custom formatter for selected sub-messages (usually based on message + type). Use to pretty print parts of the protobuf for easier diffing. + print_unknown_fields: If True, unknown fields will be printed. + force_colon: If set, a colon will be added after the field name even if the + field is a proto message. + + Returns: + str: A string of the text formatted protocol buffer message. + """ + return text_format.MessageToString( + message=message, + as_utf8=as_utf8, + as_one_line=as_one_line, + use_short_repeated_primitives=use_short_repeated_primitives, + pointy_brackets=pointy_brackets, + use_index_order=use_index_order, + float_format=float_format, + double_format=double_format, + use_field_number=use_field_number, + descriptor_pool=descriptor_pool, + indent=indent, + message_formatter=message_formatter, + print_unknown_fields=print_unknown_fields, + force_colon=force_colon, + ) + + +def parse( + message_class: Type[Message], + text: AnyStr, + allow_unknown_extension: bool = False, + allow_field_number: bool = False, + descriptor_pool: Optional[DescriptorPool] = None, + allow_unknown_field: bool = False, +) -> Message: + """Parses a text representation of a protocol message into a message. + + Args: + message_class: The message meta class. + text (str): Message text representation. + message (Message): A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types. + allow_unknown_field: if True, skip over unknown field and keep parsing. + Avoid to use this option if possible. It may hide some errors (e.g. + spelling error on field name) + + Returns: + Message: A new message passed from text. + + Raises: + ParseError: On text parsing problems. + """ + new_message = message_class() + text_format.Parse( + text=text, + message=new_message, + allow_unknown_extension=allow_unknown_extension, + allow_field_number=allow_field_number, + descriptor_pool=descriptor_pool, + allow_unknown_field=allow_unknown_field, + ) + return new_message diff --git a/lib/python3.12/site-packages/google/protobuf/pyext/__init__.py b/lib/python3.12/site-packages/google/protobuf/pyext/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/google/protobuf/pyext/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/pyext/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a1c1b6446ddd9a560eacbbb0b0a5912e894e4a9e Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/pyext/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/pyext/__pycache__/cpp_message.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/pyext/__pycache__/cpp_message.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9d484f6a22e482684932ff9c63b8e9808cc80c32 Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/pyext/__pycache__/cpp_message.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/pyext/cpp_message.py b/lib/python3.12/site-packages/google/protobuf/pyext/cpp_message.py new file mode 100644 index 0000000000000000000000000000000000000000..54cfe30646aaaed023e4b6228ad291aa7c53d147 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/pyext/cpp_message.py @@ -0,0 +1,49 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Protocol message implementation hooks for C++ implementation. + +Contains helper functions used to create protocol message classes from +Descriptor objects at runtime backed by the protocol buffer C++ API. +""" + +__author__ = 'tibell@google.com (Johan Tibell)' + +from google.protobuf.internal import api_implementation + + +# pylint: disable=protected-access +_message = api_implementation._c_module +# TODO: Remove this import after fix api_implementation +if _message is None: + from google.protobuf.pyext import _message + + +class GeneratedProtocolMessageType(_message.MessageMeta): + + """Metaclass for protocol message classes created at runtime from Descriptors. + + The protocol compiler currently uses this metaclass to create protocol + message classes at runtime. Clients can also manually create their own + classes at runtime, as in this example: + + mydescriptor = Descriptor(.....) + factory = symbol_database.Default() + factory.pool.AddDescriptor(mydescriptor) + MyProtoClass = message_factory.GetMessageClass(mydescriptor) + myproto_instance = MyProtoClass() + myproto.foo_field = 23 + ... + + The above example will not work for nested types. If you wish to include them, + use reflection.MakeClass() instead of manually instantiating the class in + order to create the appropriate class structure. + """ + + # Must be consistent with the protocol-compiler code in + # proto2/compiler/internal/generator.*. + _DESCRIPTOR_KEY = 'DESCRIPTOR' diff --git a/lib/python3.12/site-packages/google/protobuf/reflection.py b/lib/python3.12/site-packages/google/protobuf/reflection.py new file mode 100644 index 0000000000000000000000000000000000000000..a29f41a0527c78f437ecb564d8291a7485413a06 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/reflection.py @@ -0,0 +1,36 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +# This code is meant to work on Python 2.4 and above only. + +"""Contains a metaclass and helper functions used to create +protocol message classes from Descriptor objects at runtime. + +Recall that a metaclass is the "type" of a class. +(A class is to a metaclass what an instance is to a class.) + +In this case, we use the GeneratedProtocolMessageType metaclass +to inject all the useful functionality into the classes +output by the protocol compiler at compile-time. + +The upshot of all this is that the real implementation +details for ALL pure-Python protocol buffers are *here in +this file*. +""" + +__author__ = 'robinson@google.com (Will Robinson)' + +import warnings + +from google.protobuf import message_factory +from google.protobuf import symbol_database + +# The type of all Message classes. +# Part of the public interface, but normally only used by message factories. +GeneratedProtocolMessageType = message_factory._GENERATED_PROTOCOL_MESSAGE_TYPE + +MESSAGE_CLASS_CACHE = {} diff --git a/lib/python3.12/site-packages/google/protobuf/runtime_version.py b/lib/python3.12/site-packages/google/protobuf/runtime_version.py new file mode 100644 index 0000000000000000000000000000000000000000..30fbda235718394ed644362e0cd51fcfc06bfde2 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/runtime_version.py @@ -0,0 +1,104 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Protobuf Runtime versions and validators. + +It should only be accessed by Protobuf gencodes and tests. DO NOT USE it +elsewhere. +""" + +__author__ = 'shaod@google.com (Dennis Shao)' + +from enum import Enum +import os +import warnings + + +class Domain(Enum): + GOOGLE_INTERNAL = 1 + PUBLIC = 2 + + +# The versions of this Python Protobuf runtime to be changed automatically by +# the Protobuf release process. Do not edit them manually. +# These OSS versions are not stripped to avoid merging conflicts. +OSS_DOMAIN = Domain.PUBLIC +OSS_MAJOR = 6 +OSS_MINOR = 33 +OSS_PATCH = 5 +OSS_SUFFIX = '' + +DOMAIN = OSS_DOMAIN +MAJOR = OSS_MAJOR +MINOR = OSS_MINOR +PATCH = OSS_PATCH +SUFFIX = OSS_SUFFIX + +# Avoid flooding of warnings. +_MAX_WARNING_COUNT = 20 +_warning_count = 0 + +class VersionError(Exception): + """Exception class for version violation.""" + + +def _ReportVersionError(msg): + raise VersionError(msg) + + +def ValidateProtobufRuntimeVersion( + gen_domain, gen_major, gen_minor, gen_patch, gen_suffix, location +): + """Function to validate versions. + + Args: + gen_domain: The domain where the code was generated from. + gen_major: The major version number of the gencode. + gen_minor: The minor version number of the gencode. + gen_patch: The patch version number of the gencode. + gen_suffix: The version suffix e.g. '-dev', '-rc1' of the gencode. + location: The proto location that causes the version violation. + + Raises: + VersionError: if gencode version is invalid or incompatible with the + runtime. + """ + + disable_flag = os.getenv('TEMPORARILY_DISABLE_PROTOBUF_VERSION_CHECK') + if disable_flag is not None and disable_flag.lower() == 'true': + return + + global _warning_count + + version = f'{MAJOR}.{MINOR}.{PATCH}{SUFFIX}' + gen_version = f'{gen_major}.{gen_minor}.{gen_patch}{gen_suffix}' + + if gen_major < 0 or gen_minor < 0 or gen_patch < 0: + raise VersionError(f'Invalid gencode version: {gen_version}') + + error_prompt = ( + 'See Protobuf version guarantees at' + ' https://protobuf.dev/support/cross-version-runtime-guarantee.' + ) + + if gen_domain != DOMAIN: + _ReportVersionError( + 'Detected mismatched Protobuf Gencode/Runtime domains when loading' + f' {location}: gencode {gen_domain.name} runtime {DOMAIN.name}.' + ' Cross-domain usage of Protobuf is not supported.' + ) + + if ( + MAJOR < gen_major + or (MAJOR == gen_major and MINOR < gen_minor) + or (MAJOR == gen_major and MINOR == gen_minor and PATCH < gen_patch) + ): + _ReportVersionError( + 'Detected incompatible Protobuf Gencode/Runtime versions when loading' + f' {location}: gencode {gen_version} runtime {version}. Runtime version' + f' cannot be older than the linked gencode version. {error_prompt}' + ) diff --git a/lib/python3.12/site-packages/google/protobuf/service_reflection.py b/lib/python3.12/site-packages/google/protobuf/service_reflection.py new file mode 100644 index 0000000000000000000000000000000000000000..7ba3d0b2e448629bc53e1e9c75f0249a0b74c033 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/service_reflection.py @@ -0,0 +1,272 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains metaclasses used to create protocol service and service stub +classes from ServiceDescriptor objects at runtime. + +The GeneratedServiceType and GeneratedServiceStubType metaclasses are used to +inject all useful functionality into the classes output by the protocol +compiler at compile-time. +""" + +__author__ = 'petar@google.com (Petar Petrov)' + + +class GeneratedServiceType(type): + + """Metaclass for service classes created at runtime from ServiceDescriptors. + + Implementations for all methods described in the Service class are added here + by this class. We also create properties to allow getting/setting all fields + in the protocol message. + + The protocol compiler currently uses this metaclass to create protocol service + classes at runtime. Clients can also manually create their own classes at + runtime, as in this example:: + + mydescriptor = ServiceDescriptor(.....) + class MyProtoService(service.Service): + __metaclass__ = GeneratedServiceType + DESCRIPTOR = mydescriptor + myservice_instance = MyProtoService() + # ... + """ + + _DESCRIPTOR_KEY = 'DESCRIPTOR' + + def __init__(cls, name, bases, dictionary): + """Creates a message service class. + + Args: + name: Name of the class (ignored, but required by the metaclass + protocol). + bases: Base classes of the class being constructed. + dictionary: The class dictionary of the class being constructed. + dictionary[_DESCRIPTOR_KEY] must contain a ServiceDescriptor object + describing this protocol service type. + """ + # Don't do anything if this class doesn't have a descriptor. This happens + # when a service class is subclassed. + if GeneratedServiceType._DESCRIPTOR_KEY not in dictionary: + return + + descriptor = dictionary[GeneratedServiceType._DESCRIPTOR_KEY] + service_builder = _ServiceBuilder(descriptor) + service_builder.BuildService(cls) + cls.DESCRIPTOR = descriptor + + +class GeneratedServiceStubType(GeneratedServiceType): + + """Metaclass for service stubs created at runtime from ServiceDescriptors. + + This class has similar responsibilities as GeneratedServiceType, except that + it creates the service stub classes. + """ + + _DESCRIPTOR_KEY = 'DESCRIPTOR' + + def __init__(cls, name, bases, dictionary): + """Creates a message service stub class. + + Args: + name: Name of the class (ignored, here). + bases: Base classes of the class being constructed. + dictionary: The class dictionary of the class being constructed. + dictionary[_DESCRIPTOR_KEY] must contain a ServiceDescriptor object + describing this protocol service type. + """ + super(GeneratedServiceStubType, cls).__init__(name, bases, dictionary) + # Don't do anything if this class doesn't have a descriptor. This happens + # when a service stub is subclassed. + if GeneratedServiceStubType._DESCRIPTOR_KEY not in dictionary: + return + + descriptor = dictionary[GeneratedServiceStubType._DESCRIPTOR_KEY] + service_stub_builder = _ServiceStubBuilder(descriptor) + service_stub_builder.BuildServiceStub(cls) + + +class _ServiceBuilder(object): + + """This class constructs a protocol service class using a service descriptor. + + Given a service descriptor, this class constructs a class that represents + the specified service descriptor. One service builder instance constructs + exactly one service class. That means all instances of that class share the + same builder. + """ + + def __init__(self, service_descriptor): + """Initializes an instance of the service class builder. + + Args: + service_descriptor: ServiceDescriptor to use when constructing the + service class. + """ + self.descriptor = service_descriptor + + def BuildService(builder, cls): + """Constructs the service class. + + Args: + cls: The class that will be constructed. + """ + + # CallMethod needs to operate with an instance of the Service class. This + # internal wrapper function exists only to be able to pass the service + # instance to the method that does the real CallMethod work. + # Making sure to use exact argument names from the abstract interface in + # service.py to match the type signature + def _WrapCallMethod(self, method_descriptor, rpc_controller, request, done): + return builder._CallMethod(self, method_descriptor, rpc_controller, + request, done) + + def _WrapGetRequestClass(self, method_descriptor): + return builder._GetRequestClass(method_descriptor) + + def _WrapGetResponseClass(self, method_descriptor): + return builder._GetResponseClass(method_descriptor) + + builder.cls = cls + cls.CallMethod = _WrapCallMethod + cls.GetDescriptor = staticmethod(lambda: builder.descriptor) + cls.GetDescriptor.__doc__ = 'Returns the service descriptor.' + cls.GetRequestClass = _WrapGetRequestClass + cls.GetResponseClass = _WrapGetResponseClass + for method in builder.descriptor.methods: + setattr(cls, method.name, builder._GenerateNonImplementedMethod(method)) + + def _CallMethod(self, srvc, method_descriptor, + rpc_controller, request, callback): + """Calls the method described by a given method descriptor. + + Args: + srvc: Instance of the service for which this method is called. + method_descriptor: Descriptor that represent the method to call. + rpc_controller: RPC controller to use for this method's execution. + request: Request protocol message. + callback: A callback to invoke after the method has completed. + """ + if method_descriptor.containing_service != self.descriptor: + raise RuntimeError( + 'CallMethod() given method descriptor for wrong service type.') + method = getattr(srvc, method_descriptor.name) + return method(rpc_controller, request, callback) + + def _GetRequestClass(self, method_descriptor): + """Returns the class of the request protocol message. + + Args: + method_descriptor: Descriptor of the method for which to return the + request protocol message class. + + Returns: + A class that represents the input protocol message of the specified + method. + """ + if method_descriptor.containing_service != self.descriptor: + raise RuntimeError( + 'GetRequestClass() given method descriptor for wrong service type.') + return method_descriptor.input_type._concrete_class + + def _GetResponseClass(self, method_descriptor): + """Returns the class of the response protocol message. + + Args: + method_descriptor: Descriptor of the method for which to return the + response protocol message class. + + Returns: + A class that represents the output protocol message of the specified + method. + """ + if method_descriptor.containing_service != self.descriptor: + raise RuntimeError( + 'GetResponseClass() given method descriptor for wrong service type.') + return method_descriptor.output_type._concrete_class + + def _GenerateNonImplementedMethod(self, method): + """Generates and returns a method that can be set for a service methods. + + Args: + method: Descriptor of the service method for which a method is to be + generated. + + Returns: + A method that can be added to the service class. + """ + return lambda inst, rpc_controller, request, callback: ( + self._NonImplementedMethod(method.name, rpc_controller, callback)) + + def _NonImplementedMethod(self, method_name, rpc_controller, callback): + """The body of all methods in the generated service class. + + Args: + method_name: Name of the method being executed. + rpc_controller: RPC controller used to execute this method. + callback: A callback which will be invoked when the method finishes. + """ + rpc_controller.SetFailed('Method %s not implemented.' % method_name) + callback(None) + + +class _ServiceStubBuilder(object): + + """Constructs a protocol service stub class using a service descriptor. + + Given a service descriptor, this class constructs a suitable stub class. + A stub is just a type-safe wrapper around an RpcChannel which emulates a + local implementation of the service. + + One service stub builder instance constructs exactly one class. It means all + instances of that class share the same service stub builder. + """ + + def __init__(self, service_descriptor): + """Initializes an instance of the service stub class builder. + + Args: + service_descriptor: ServiceDescriptor to use when constructing the + stub class. + """ + self.descriptor = service_descriptor + + def BuildServiceStub(self, cls): + """Constructs the stub class. + + Args: + cls: The class that will be constructed. + """ + + def _ServiceStubInit(stub, rpc_channel): + stub.rpc_channel = rpc_channel + self.cls = cls + cls.__init__ = _ServiceStubInit + for method in self.descriptor.methods: + setattr(cls, method.name, self._GenerateStubMethod(method)) + + def _GenerateStubMethod(self, method): + return (lambda inst, rpc_controller, request, callback=None: + self._StubMethod(inst, method, rpc_controller, request, callback)) + + def _StubMethod(self, stub, method_descriptor, + rpc_controller, request, callback): + """The body of all service methods in the generated stub class. + + Args: + stub: Stub instance. + method_descriptor: Descriptor of the invoked method. + rpc_controller: Rpc controller to execute the method. + request: Request protocol message. + callback: A callback to execute when the method finishes. + Returns: + Response message (in case of blocking call). + """ + return stub.rpc_channel.CallMethod( + method_descriptor, rpc_controller, request, + method_descriptor.output_type._concrete_class, callback) diff --git a/lib/python3.12/site-packages/google/protobuf/source_context_pb2.py b/lib/python3.12/site-packages/google/protobuf/source_context_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..38014fea8ca0c2600b0912f3e2ff89a22109f65b --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/source_context_pb2.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/source_context.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/source_context.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$google/protobuf/source_context.proto\x12\x0fgoogle.protobuf\",\n\rSourceContext\x12\x1b\n\tfile_name\x18\x01 \x01(\tR\x08\x66ileNameB\x8a\x01\n\x13\x63om.google.protobufB\x12SourceContextProtoP\x01Z6google.golang.org/protobuf/types/known/sourcecontextpb\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.source_context_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\022SourceContextProtoP\001Z6google.golang.org/protobuf/types/known/sourcecontextpb\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_SOURCECONTEXT']._serialized_start=57 + _globals['_SOURCECONTEXT']._serialized_end=101 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/struct_pb2.py b/lib/python3.12/site-packages/google/protobuf/struct_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..4dff6dd4583d434e36396d91999f448f70d34f2f --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/struct_pb2.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/struct.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/struct.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1cgoogle/protobuf/struct.proto\x12\x0fgoogle.protobuf\"\x98\x01\n\x06Struct\x12;\n\x06\x66ields\x18\x01 \x03(\x0b\x32#.google.protobuf.Struct.FieldsEntryR\x06\x66ields\x1aQ\n\x0b\x46ieldsEntry\x12\x10\n\x03key\x18\x01 \x01(\tR\x03key\x12,\n\x05value\x18\x02 \x01(\x0b\x32\x16.google.protobuf.ValueR\x05value:\x02\x38\x01\"\xb2\x02\n\x05Value\x12;\n\nnull_value\x18\x01 \x01(\x0e\x32\x1a.google.protobuf.NullValueH\x00R\tnullValue\x12#\n\x0cnumber_value\x18\x02 \x01(\x01H\x00R\x0bnumberValue\x12#\n\x0cstring_value\x18\x03 \x01(\tH\x00R\x0bstringValue\x12\x1f\n\nbool_value\x18\x04 \x01(\x08H\x00R\tboolValue\x12<\n\x0cstruct_value\x18\x05 \x01(\x0b\x32\x17.google.protobuf.StructH\x00R\x0bstructValue\x12;\n\nlist_value\x18\x06 \x01(\x0b\x32\x1a.google.protobuf.ListValueH\x00R\tlistValueB\x06\n\x04kind\";\n\tListValue\x12.\n\x06values\x18\x01 \x03(\x0b\x32\x16.google.protobuf.ValueR\x06values*\x1b\n\tNullValue\x12\x0e\n\nNULL_VALUE\x10\x00\x42\x7f\n\x13\x63om.google.protobufB\x0bStructProtoP\x01Z/google.golang.org/protobuf/types/known/structpb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.struct_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\013StructProtoP\001Z/google.golang.org/protobuf/types/known/structpb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_STRUCT_FIELDSENTRY']._loaded_options = None + _globals['_STRUCT_FIELDSENTRY']._serialized_options = b'8\001' + _globals['_NULLVALUE']._serialized_start=574 + _globals['_NULLVALUE']._serialized_end=601 + _globals['_STRUCT']._serialized_start=50 + _globals['_STRUCT']._serialized_end=202 + _globals['_STRUCT_FIELDSENTRY']._serialized_start=121 + _globals['_STRUCT_FIELDSENTRY']._serialized_end=202 + _globals['_VALUE']._serialized_start=205 + _globals['_VALUE']._serialized_end=511 + _globals['_LISTVALUE']._serialized_start=513 + _globals['_LISTVALUE']._serialized_end=572 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/symbol_database.py b/lib/python3.12/site-packages/google/protobuf/symbol_database.py new file mode 100644 index 0000000000000000000000000000000000000000..a538d5d6da750ed4b48d14689b5f3d894a263538 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/symbol_database.py @@ -0,0 +1,179 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""A database of Python protocol buffer generated symbols. + +SymbolDatabase is the MessageFactory for messages generated at compile time, +and makes it easy to create new instances of a registered type, given only the +type's protocol buffer symbol name. + +Example usage:: + + db = symbol_database.SymbolDatabase() + + # Register symbols of interest, from one or multiple files. + db.RegisterFileDescriptor(my_proto_pb2.DESCRIPTOR) + db.RegisterMessage(my_proto_pb2.MyMessage) + db.RegisterEnumDescriptor(my_proto_pb2.MyEnum.DESCRIPTOR) + + # The database can be used as a MessageFactory, to generate types based on + # their name: + types = db.GetMessages(['my_proto.proto']) + my_message_instance = types['MyMessage']() + + # The database's underlying descriptor pool can be queried, so it's not + # necessary to know a type's filename to be able to generate it: + filename = db.pool.FindFileContainingSymbol('MyMessage') + my_message_instance = db.GetMessages([filename])['MyMessage']() + + # This functionality is also provided directly via a convenience method: + my_message_instance = db.GetSymbol('MyMessage')() +""" + +import warnings + +from google.protobuf.internal import api_implementation +from google.protobuf import descriptor_pool +from google.protobuf import message_factory + + +class SymbolDatabase(): + """A database of Python generated symbols.""" + + # local cache of registered classes. + _classes = {} + + def __init__(self, pool=None): + """Initializes a new SymbolDatabase.""" + self.pool = pool or descriptor_pool.DescriptorPool() + + def RegisterMessage(self, message): + """Registers the given message type in the local database. + + Calls to GetSymbol() and GetMessages() will return messages registered here. + + Args: + message: A :class:`google.protobuf.message.Message` subclass (or + instance); its descriptor will be registered. + + Returns: + The provided message. + """ + + desc = message.DESCRIPTOR + self._classes[desc] = message + self.RegisterMessageDescriptor(desc) + return message + + def RegisterMessageDescriptor(self, message_descriptor): + """Registers the given message descriptor in the local database. + + Args: + message_descriptor (Descriptor): the message descriptor to add. + """ + if api_implementation.Type() == 'python': + # pylint: disable=protected-access + self.pool._AddDescriptor(message_descriptor) + + def RegisterEnumDescriptor(self, enum_descriptor): + """Registers the given enum descriptor in the local database. + + Args: + enum_descriptor (EnumDescriptor): The enum descriptor to register. + + Returns: + EnumDescriptor: The provided descriptor. + """ + if api_implementation.Type() == 'python': + # pylint: disable=protected-access + self.pool._AddEnumDescriptor(enum_descriptor) + return enum_descriptor + + def RegisterServiceDescriptor(self, service_descriptor): + """Registers the given service descriptor in the local database. + + Args: + service_descriptor (ServiceDescriptor): the service descriptor to + register. + """ + if api_implementation.Type() == 'python': + # pylint: disable=protected-access + self.pool._AddServiceDescriptor(service_descriptor) + + def RegisterFileDescriptor(self, file_descriptor): + """Registers the given file descriptor in the local database. + + Args: + file_descriptor (FileDescriptor): The file descriptor to register. + """ + if api_implementation.Type() == 'python': + # pylint: disable=protected-access + self.pool._InternalAddFileDescriptor(file_descriptor) + + def GetSymbol(self, symbol): + """Tries to find a symbol in the local database. + + Currently, this method only returns message.Message instances, however, if + may be extended in future to support other symbol types. + + Args: + symbol (str): a protocol buffer symbol. + + Returns: + A Python class corresponding to the symbol. + + Raises: + KeyError: if the symbol could not be found. + """ + + return self._classes[self.pool.FindMessageTypeByName(symbol)] + + def GetMessages(self, files): + # TODO: Fix the differences with MessageFactory. + """Gets all registered messages from a specified file. + + Only messages already created and registered will be returned; (this is the + case for imported _pb2 modules) + But unlike MessageFactory, this version also returns already defined nested + messages, but does not register any message extensions. + + Args: + files (list[str]): The file names to extract messages from. + + Returns: + A dictionary mapping proto names to the message classes. + + Raises: + KeyError: if a file could not be found. + """ + + def _GetAllMessages(desc): + """Walk a message Descriptor and recursively yields all message names.""" + yield desc + for msg_desc in desc.nested_types: + for nested_desc in _GetAllMessages(msg_desc): + yield nested_desc + + result = {} + for file_name in files: + file_desc = self.pool.FindFileByName(file_name) + for msg_desc in file_desc.message_types_by_name.values(): + for desc in _GetAllMessages(msg_desc): + try: + result[desc.full_name] = self._classes[desc] + except KeyError: + # This descriptor has no registered class, skip it. + pass + return result + + +_DEFAULT = SymbolDatabase(pool=descriptor_pool.Default()) + + +def Default(): + """Returns the default SymbolDatabase.""" + return _DEFAULT diff --git a/lib/python3.12/site-packages/google/protobuf/testdata/__init__.py b/lib/python3.12/site-packages/google/protobuf/testdata/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/google/protobuf/testdata/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/testdata/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ae427e7a343ed36b39bc6cce4be3fd183b0c6dda Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/testdata/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/text_encoding.py b/lib/python3.12/site-packages/google/protobuf/text_encoding.py new file mode 100644 index 0000000000000000000000000000000000000000..03c27dc104b2b9b9ab708084d9a5480bb9d058b6 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/text_encoding.py @@ -0,0 +1,106 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Encoding related utilities.""" +import re + +def _AsciiIsPrint(i): + return i >= 32 and i < 127 + +def _MakeStrEscapes(): + ret = {} + for i in range(0, 128): + if not _AsciiIsPrint(i): + ret[i] = r'\%03o' % i + ret[ord('\t')] = r'\t' # optional escape + ret[ord('\n')] = r'\n' # optional escape + ret[ord('\r')] = r'\r' # optional escape + ret[ord('"')] = r'\"' # necessary escape + ret[ord('\'')] = r"\'" # optional escape + ret[ord('\\')] = r'\\' # necessary escape + return ret + +# Maps int -> char, performing string escapes. +_str_escapes = _MakeStrEscapes() + +# Maps int -> char, performing byte escaping and string escapes +_byte_escapes = {i: chr(i) for i in range(0, 256)} +_byte_escapes.update(_str_escapes) +_byte_escapes.update({i: r'\%03o' % i for i in range(128, 256)}) + + +def _DecodeUtf8EscapeErrors(text_bytes): + ret = '' + while text_bytes: + try: + ret += text_bytes.decode('utf-8').translate(_str_escapes) + text_bytes = '' + except UnicodeDecodeError as e: + ret += text_bytes[:e.start].decode('utf-8').translate(_str_escapes) + ret += _byte_escapes[text_bytes[e.start]] + text_bytes = text_bytes[e.start+1:] + return ret + + +def CEscape(text, as_utf8) -> str: + """Escape a bytes string for use in an text protocol buffer. + + Args: + text: A byte string to be escaped. + as_utf8: Specifies if result may contain non-ASCII characters. + In Python 3 this allows unescaped non-ASCII Unicode characters. + In Python 2 the return value will be valid UTF-8 rather than only ASCII. + Returns: + Escaped string (str). + """ + # Python's text.encode() 'string_escape' or 'unicode_escape' codecs do not + # satisfy our needs; they encodes unprintable characters using two-digit hex + # escapes whereas our C++ unescaping function allows hex escapes to be any + # length. So, "\0011".encode('string_escape') ends up being "\\x011", which + # will be decoded in C++ as a single-character string with char code 0x11. + text_is_unicode = isinstance(text, str) + if as_utf8: + if text_is_unicode: + return text.translate(_str_escapes) + else: + return _DecodeUtf8EscapeErrors(text) + else: + if text_is_unicode: + text = text.encode('utf-8') + return ''.join([_byte_escapes[c] for c in text]) + + +_CUNESCAPE_HEX = re.compile(r'(\\+)x([0-9a-fA-F])(?![0-9a-fA-F])') + + +def CUnescape(text: str) -> bytes: + """Unescape a text string with C-style escape sequences to UTF-8 bytes. + + Args: + text: The data to parse in a str. + Returns: + A byte string. + """ + + def ReplaceHex(m): + # Only replace the match if the number of leading back slashes is odd. i.e. + # the slash itself is not escaped. + if len(m.group(1)) & 1: + return m.group(1) + 'x0' + m.group(2) + return m.group(0) + + # This is required because the 'string_escape' encoding doesn't + # allow single-digit hex escapes (like '\xf'). + result = _CUNESCAPE_HEX.sub(ReplaceHex, text) + + # Replaces Unicode escape sequences with their character equivalents. + result = result.encode('raw_unicode_escape').decode('raw_unicode_escape') + # Encode Unicode characters as UTF-8, then decode to Latin-1 escaping + # unprintable characters. + result = result.encode('utf-8').decode('unicode_escape') + # Convert Latin-1 text back to a byte string (latin-1 codec also works here). + return result.encode('latin-1') diff --git a/lib/python3.12/site-packages/google/protobuf/text_format.py b/lib/python3.12/site-packages/google/protobuf/text_format.py new file mode 100644 index 0000000000000000000000000000000000000000..a3566ee3ded95587eff8263ab95efa9427f6ae1e --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/text_format.py @@ -0,0 +1,1884 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains routines for printing protocol messages in text format. + +Simple usage example:: + + # Create a proto object and serialize it to a text proto string. + message = my_proto_pb2.MyMessage(foo='bar') + text_proto = text_format.MessageToString(message) + + # Parse a text proto string. + message = text_format.Parse(text_proto, my_proto_pb2.MyMessage()) +""" + +__author__ = 'kenton@google.com (Kenton Varda)' + +# TODO Import thread contention leads to test failures. +import encodings.raw_unicode_escape # pylint: disable=unused-import +import encodings.unicode_escape # pylint: disable=unused-import +import io +import math +import re +import warnings + +from google.protobuf.internal import decoder +from google.protobuf.internal import type_checkers +from google.protobuf import descriptor +from google.protobuf import text_encoding +from google.protobuf import unknown_fields + +# pylint: disable=g-import-not-at-top +__all__ = ['MessageToString', 'Parse', 'PrintMessage', 'PrintField', + 'PrintFieldValue', 'Merge', 'MessageToBytes'] + +_INTEGER_CHECKERS = (type_checkers.Uint32ValueChecker(), + type_checkers.Int32ValueChecker(), + type_checkers.Uint64ValueChecker(), + type_checkers.Int64ValueChecker()) +_FLOAT_INFINITY = re.compile('-?inf(?:inity)?f?$', re.IGNORECASE) +_FLOAT_NAN = re.compile('nanf?$', re.IGNORECASE) +_FLOAT_OCTAL_PREFIX = re.compile('-?0[0-9]+') +_QUOTES = frozenset(("'", '"')) +_ANY_FULL_TYPE_NAME = 'google.protobuf.Any' +_DEBUG_STRING_SILENT_MARKER = '\t ' + +_as_utf8_default = True + + +class Error(Exception): + """Top-level module error for text_format.""" + + +class ParseError(Error): + """Thrown in case of text parsing or tokenizing error.""" + + def __init__(self, message=None, line=None, column=None): + if message is not None and line is not None: + loc = str(line) + if column is not None: + loc += ':{0}'.format(column) + message = '{0} : {1}'.format(loc, message) + if message is not None: + super(ParseError, self).__init__(message) + else: + super(ParseError, self).__init__() + self._line = line + self._column = column + + def GetLine(self): + return self._line + + def GetColumn(self): + return self._column + + +class TextWriter(object): + + def __init__(self, as_utf8): + self._writer = io.StringIO() + + def write(self, val): + return self._writer.write(val) + + def close(self): + return self._writer.close() + + def getvalue(self): + return self._writer.getvalue() + + +def MessageToString( + message, + as_utf8=_as_utf8_default, + as_one_line=False, + use_short_repeated_primitives=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + double_format=None, + use_field_number=False, + descriptor_pool=None, + indent=0, + message_formatter=None, + print_unknown_fields=False, + force_colon=False) -> str: + """Convert protobuf message to text format. + + Double values can be formatted compactly with 15 digits of + precision (which is the most that IEEE 754 "double" can guarantee) + using double_format='.15g'. To ensure that converting to text and back to a + proto will result in an identical value, double_format='.17g' should be used. + + Args: + message: The protocol buffers message. + as_utf8: Return unescaped Unicode for non-ASCII characters. + as_one_line: Don't introduce newlines between fields. + use_short_repeated_primitives: Use short repeated format for primitives. + pointy_brackets: If True, use angle brackets instead of curly braces for + nesting. + use_index_order: If True, fields of a proto message will be printed using + the order defined in source code instead of the field number, extensions + will be printed at the end of the message and their relative order is + determined by the extension number. By default, use the field number + order. + float_format (str): Deprecated. If set, use this to specify float field + formatting (per the "Format Specification Mini-Language"); otherwise, + shortest float that has same value in wire will be printed. Also affect + double field if double_format is not set but float_format is set. + double_format (str): Deprecated. If set, use this to specify double field + formatting (per the "Format Specification Mini-Language"); if it is not + set but float_format is set, use float_format. Otherwise, use ``str()`` + use_field_number: If True, print field numbers instead of names. + descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types. + indent (int): The initial indent level, in terms of spaces, for pretty + print. + message_formatter (function(message, indent, as_one_line) -> unicode|None): + Custom formatter for selected sub-messages (usually based on message + type). Use to pretty print parts of the protobuf for easier diffing. + print_unknown_fields: If True, unknown fields will be printed. + force_colon: If set, a colon will be added after the field name even if the + field is a proto message. + + Returns: + str: A string of the text formatted protocol buffer message. + """ + out = TextWriter(as_utf8) + printer = _Printer( + out, + indent, + as_utf8, + as_one_line, + use_short_repeated_primitives, + pointy_brackets, + use_index_order, + float_format, + double_format, + use_field_number, + descriptor_pool, + message_formatter, + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) + printer.PrintMessage(message) + result = out.getvalue() + out.close() + if as_one_line: + return result.rstrip() + return result + + +def MessageToBytes(message, **kwargs) -> bytes: + """Convert protobuf message to encoded text format. See MessageToString.""" + text = MessageToString(message, **kwargs) + if isinstance(text, bytes): + return text + codec = 'utf-8' if kwargs.get('as_utf8') else 'ascii' + return text.encode(codec) + + +def _IsMapEntry(field): + return (field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + field.message_type.has_options and + field.message_type.GetOptions().map_entry) + + +def _IsGroupLike(field): + """Determines if a field is consistent with a proto2 group. + + Args: + field: The field descriptor. + + Returns: + True if this field is group-like, false otherwise. + """ + # Groups are always tag-delimited. + if field.type != descriptor.FieldDescriptor.TYPE_GROUP: + return False + + # Group fields always are always the lowercase type name. + if field.name != field.message_type.name.lower(): + return False + + if field.message_type.file != field.file: + return False + + # Group messages are always defined in the same scope as the field. File + # level extensions will compare NULL == NULL here, which is why the file + # comparison above is necessary to ensure both come from the same file. + return ( + field.message_type.containing_type == field.extension_scope + if field.is_extension + else field.message_type.containing_type == field.containing_type + ) + + +def PrintMessage(message, + out, + indent=0, + as_utf8=_as_utf8_default, + as_one_line=False, + use_short_repeated_primitives=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + double_format=None, + use_field_number=False, + descriptor_pool=None, + message_formatter=None, + print_unknown_fields=False, + force_colon=False): + """Convert the message to text format and write it to the out stream. + + Args: + message: The Message object to convert to text format. + out: A file handle to write the message to. + indent: The initial indent level for pretty print. + as_utf8: Return unescaped Unicode for non-ASCII characters. + as_one_line: Don't introduce newlines between fields. + use_short_repeated_primitives: Use short repeated format for primitives. + pointy_brackets: If True, use angle brackets instead of curly braces for + nesting. + use_index_order: If True, print fields of a proto message using the order + defined in source code instead of the field number. By default, use the + field number order. + float_format: If set, use this to specify float field formatting + (per the "Format Specification Mini-Language"); otherwise, shortest + float that has same value in wire will be printed. Also affect double + field if double_format is not set but float_format is set. + double_format: If set, use this to specify double field formatting + (per the "Format Specification Mini-Language"); if it is not set but + float_format is set, use float_format. Otherwise, str() is used. + use_field_number: If True, print field numbers instead of names. + descriptor_pool: A DescriptorPool used to resolve Any types. + message_formatter: A function(message, indent, as_one_line): unicode|None + to custom format selected sub-messages (usually based on message type). + Use to pretty print parts of the protobuf for easier diffing. + print_unknown_fields: If True, unknown fields will be printed. + force_colon: If set, a colon will be added after the field name even if + the field is a proto message. + """ + printer = _Printer( + out=out, indent=indent, as_utf8=as_utf8, + as_one_line=as_one_line, + use_short_repeated_primitives=use_short_repeated_primitives, + pointy_brackets=pointy_brackets, + use_index_order=use_index_order, + float_format=float_format, + double_format=double_format, + use_field_number=use_field_number, + descriptor_pool=descriptor_pool, + message_formatter=message_formatter, + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) + printer.PrintMessage(message) + + +def PrintField(field, + value, + out, + indent=0, + as_utf8=_as_utf8_default, + as_one_line=False, + use_short_repeated_primitives=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + double_format=None, + message_formatter=None, + print_unknown_fields=False, + force_colon=False): + """Print a single field name/value pair.""" + printer = _Printer(out, indent, as_utf8, as_one_line, + use_short_repeated_primitives, pointy_brackets, + use_index_order, float_format, double_format, + message_formatter=message_formatter, + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) + printer.PrintField(field, value) + + +def PrintFieldValue(field, + value, + out, + indent=0, + as_utf8=_as_utf8_default, + as_one_line=False, + use_short_repeated_primitives=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + double_format=None, + message_formatter=None, + print_unknown_fields=False, + force_colon=False): + """Print a single field value (not including name).""" + printer = _Printer(out, indent, as_utf8, as_one_line, + use_short_repeated_primitives, pointy_brackets, + use_index_order, float_format, double_format, + message_formatter=message_formatter, + print_unknown_fields=print_unknown_fields, + force_colon=force_colon) + printer.PrintFieldValue(field, value) + + +def _BuildMessageFromTypeName(type_name, descriptor_pool): + """Returns a protobuf message instance. + + Args: + type_name: Fully-qualified protobuf message type name string. + descriptor_pool: DescriptorPool instance. + + Returns: + A Message instance of type matching type_name, or None if the a Descriptor + wasn't found matching type_name. + """ + # pylint: disable=g-import-not-at-top + if descriptor_pool is None: + from google.protobuf import descriptor_pool as pool_mod + descriptor_pool = pool_mod.Default() + from google.protobuf import message_factory + try: + message_descriptor = descriptor_pool.FindMessageTypeByName(type_name) + except KeyError: + return None + message_type = message_factory.GetMessageClass(message_descriptor) + return message_type() + + +# These values must match WireType enum in //google/protobuf/wire_format.h. +WIRETYPE_LENGTH_DELIMITED = 2 +WIRETYPE_START_GROUP = 3 + + +class _Printer(object): + """Text format printer for protocol message.""" + + def __init__( + self, + out, + indent=0, + as_utf8=_as_utf8_default, + as_one_line=False, + use_short_repeated_primitives=False, + pointy_brackets=False, + use_index_order=False, + float_format=None, + double_format=None, + use_field_number=False, + descriptor_pool=None, + message_formatter=None, + print_unknown_fields=False, + force_colon=False): + """Initialize the Printer. + + Double values can be formatted compactly with 15 digits of precision + (which is the most that IEEE 754 "double" can guarantee) using + double_format='.15g'. To ensure that converting to text and back to a proto + will result in an identical value, double_format='.17g' should be used. + + Args: + out: To record the text format result. + indent: The initial indent level for pretty print. + as_utf8: Return unescaped Unicode for non-ASCII characters. + as_one_line: Don't introduce newlines between fields. + use_short_repeated_primitives: Use short repeated format for primitives. + pointy_brackets: If True, use angle brackets instead of curly braces for + nesting. + use_index_order: If True, print fields of a proto message using the order + defined in source code instead of the field number. By default, use the + field number order. + float_format: Deprecated. If set, use this to specify float field + formatting (per the "Format Specification Mini-Language"); otherwise, + shortest float that has same value in wire will be printed. Also affect + double field if double_format is not set but float_format is set. + double_format: Deprecated. If set, use this to specify double field + formatting (per the "Format Specification Mini-Language"); if it is not + set but float_format is set, use float_format. Otherwise, str() is used. + use_field_number: If True, print field numbers instead of names. + descriptor_pool: A DescriptorPool used to resolve Any types. + message_formatter: A function(message, indent, as_one_line): unicode|None + to custom format selected sub-messages (usually based on message type). + Use to pretty print parts of the protobuf for easier diffing. + print_unknown_fields: If True, unknown fields will be printed. + force_colon: If set, a colon will be added after the field name even if + the field is a proto message. + """ + self.out = out + self.indent = indent + self.as_utf8 = as_utf8 + self.as_one_line = as_one_line + self.use_short_repeated_primitives = use_short_repeated_primitives + self.pointy_brackets = pointy_brackets + self.use_index_order = use_index_order + self.float_format = float_format + if double_format is not None: + warnings.warn( + 'double_format is deprecated for text_format. This will ' + 'turn into error in 7.34.0, please remove it before that.' + ) + self.double_format = double_format + else: + self.double_format = float_format + self.use_field_number = use_field_number + self.descriptor_pool = descriptor_pool + self.message_formatter = message_formatter + self.print_unknown_fields = print_unknown_fields + self.force_colon = force_colon + + def _TryPrintAsAnyMessage(self, message): + """Serializes if message is a google.protobuf.Any field.""" + if '/' not in message.type_url: + return False + packed_message = _BuildMessageFromTypeName(message.TypeName(), + self.descriptor_pool) + if packed_message is not None: + packed_message.MergeFromString(message.value) + colon = ':' if self.force_colon else '' + self.out.write('%s[%s]%s ' % (self.indent * ' ', message.type_url, colon)) + self._PrintMessageFieldValue(packed_message) + self.out.write(' ' if self.as_one_line else '\n') + return True + else: + return False + + def _TryCustomFormatMessage(self, message): + formatted = self.message_formatter(message, self.indent, self.as_one_line) + if formatted is None: + return False + + out = self.out + out.write(' ' * self.indent) + out.write(formatted) + out.write(' ' if self.as_one_line else '\n') + return True + + def PrintMessage(self, message): + """Convert protobuf message to text format. + + Args: + message: The protocol buffers message. + """ + if self.message_formatter and self._TryCustomFormatMessage(message): + return + if (message.DESCRIPTOR.full_name == _ANY_FULL_TYPE_NAME and + self._TryPrintAsAnyMessage(message)): + return + fields = message.ListFields() + if self.use_index_order: + fields.sort( + key=lambda x: x[0].number if x[0].is_extension else x[0].index) + for field, value in fields: + if _IsMapEntry(field): + for key in sorted(value): + # This is slow for maps with submessage entries because it copies the + # entire tree. Unfortunately this would take significant refactoring + # of this file to work around. + # + # TODO: refactor and optimize if this becomes an issue. + entry_submsg = value.GetEntryClass()(key=key, value=value[key]) + self.PrintField(field, entry_submsg) + elif field.is_repeated: + if (self.use_short_repeated_primitives + and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE + and field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_STRING): + self._PrintShortRepeatedPrimitivesValue(field, value) + else: + for element in value: + self.PrintField(field, element) + else: + self.PrintField(field, value) + + if self.print_unknown_fields: + self._PrintUnknownFields(unknown_fields.UnknownFieldSet(message)) + + def _PrintUnknownFields(self, unknown_field_set): + """Print unknown fields.""" + out = self.out + for field in unknown_field_set: + out.write(' ' * self.indent) + out.write(str(field.field_number)) + if field.wire_type == WIRETYPE_START_GROUP: + if self.as_one_line: + out.write(' { ') + else: + out.write(' {\n') + self.indent += 2 + + self._PrintUnknownFields(field.data) + + if self.as_one_line: + out.write('} ') + else: + self.indent -= 2 + out.write(' ' * self.indent + '}\n') + elif field.wire_type == WIRETYPE_LENGTH_DELIMITED: + try: + # If this field is parseable as a Message, it is probably + # an embedded message. + # pylint: disable=protected-access + (embedded_unknown_message, pos) = decoder._DecodeUnknownFieldSet( + memoryview(field.data), 0, len(field.data)) + except Exception: # pylint: disable=broad-except + pos = 0 + + if pos == len(field.data): + if self.as_one_line: + out.write(' { ') + else: + out.write(' {\n') + self.indent += 2 + + self._PrintUnknownFields(embedded_unknown_message) + + if self.as_one_line: + out.write('} ') + else: + self.indent -= 2 + out.write(' ' * self.indent + '}\n') + else: + # A string or bytes field. self.as_utf8 may not work. + out.write(': \"') + out.write(text_encoding.CEscape(field.data, False)) + out.write('\" ' if self.as_one_line else '\"\n') + else: + # varint, fixed32, fixed64 + out.write(': ') + out.write(str(field.data)) + out.write(' ' if self.as_one_line else '\n') + + def _PrintFieldName(self, field): + """Print field name.""" + out = self.out + out.write(' ' * self.indent) + if self.use_field_number: + out.write(str(field.number)) + else: + if field.is_extension: + out.write('[') + if (field.containing_type.GetOptions().message_set_wire_format and + field.type == descriptor.FieldDescriptor.TYPE_MESSAGE and + not field.is_required and + not field.is_repeated): + out.write(field.message_type.full_name) + else: + out.write(field.full_name) + out.write(']') + elif _IsGroupLike(field): + # For groups, use the capitalized name. + out.write(field.message_type.name) + else: + out.write(field.name) + + if (self.force_colon or + field.cpp_type != descriptor.FieldDescriptor.CPPTYPE_MESSAGE): + # The colon is optional in this case, but our cross-language golden files + # don't include it. Here, the colon is only included if force_colon is + # set to True + out.write(':') + + def PrintField(self, field, value): + """Print a single field name/value pair.""" + self._PrintFieldName(field) + self.out.write(' ') + self.PrintFieldValue(field, value) + self.out.write(' ' if self.as_one_line else '\n') + + def _PrintShortRepeatedPrimitivesValue(self, field, value): + """"Prints short repeated primitives value.""" + # Note: this is called only when value has at least one element. + self._PrintFieldName(field) + self.out.write(' [') + for i in range(len(value) - 1): + self.PrintFieldValue(field, value[i]) + self.out.write(', ') + self.PrintFieldValue(field, value[-1]) + self.out.write(']') + self.out.write(' ' if self.as_one_line else '\n') + + def _PrintMessageFieldValue(self, value): + if self.pointy_brackets: + openb = '<' + closeb = '>' + else: + openb = '{' + closeb = '}' + + if self.as_one_line: + self.out.write('%s ' % openb) + self.PrintMessage(value) + self.out.write(closeb) + else: + self.out.write('%s\n' % openb) + self.indent += 2 + self.PrintMessage(value) + self.indent -= 2 + self.out.write(' ' * self.indent + closeb) + + def PrintFieldValue(self, field, value): + """Print a single field value (not including name). + + For repeated fields, the value should be a single element. + + Args: + field: The descriptor of the field to be printed. + value: The value of the field. + """ + out = self.out + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + self._PrintMessageFieldValue(value) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_ENUM: + enum_value = field.enum_type.values_by_number.get(value, None) + if enum_value is not None: + out.write(enum_value.name) + else: + out.write(str(value)) + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_STRING: + out.write('\"') + if isinstance(value, str) and not self.as_utf8: + out_value = value.encode('utf-8') + else: + out_value = value + if field.type == descriptor.FieldDescriptor.TYPE_BYTES: + # We always need to escape all binary data in TYPE_BYTES fields. + out_as_utf8 = False + else: + out_as_utf8 = self.as_utf8 + out.write(text_encoding.CEscape(out_value, out_as_utf8)) + out.write('\"') + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_BOOL: + if value: + out.write('true') + else: + out.write('false') + elif field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_FLOAT: + if self.float_format is not None: + warnings.warn( + 'float_format is deprecated for text_format. This ' + 'will turn into error in 7.34.0, please remove it ' + 'before that.' + ) + out.write('{1:{0}}'.format(self.float_format, value)) + else: + if math.isnan(value): + out.write(str(value)) + else: + out.write(str(type_checkers.ToShortestFloat(value))) + elif (field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_DOUBLE and + self.double_format is not None): + out.write('{1:{0}}'.format(self.double_format, value)) + else: + out.write(str(value)) + + +def Parse(text, + message, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None, + allow_unknown_field=False): + """Parses a text representation of a protocol message into a message. + + NOTE: for historical reasons this function does not clear the input + message. This is different from what the binary msg.ParseFrom(...) does. + If text contains a field already set in message, the value is appended if the + field is repeated. Otherwise, an error is raised. + + Example:: + + a = MyProto() + a.repeated_field.append('test') + b = MyProto() + + # Repeated fields are combined + text_format.Parse(repr(a), b) + text_format.Parse(repr(a), b) # repeated_field contains ["test", "test"] + + # Non-repeated fields cannot be overwritten + a.singular_field = 1 + b.singular_field = 2 + text_format.Parse(repr(a), b) # ParseError + + # Binary version: + b.ParseFromString(a.SerializeToString()) # repeated_field is now "test" + + Caller is responsible for clearing the message as needed. + + Args: + text (str): Message text representation. + message (Message): A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types. + allow_unknown_field: if True, skip over unknown field and keep + parsing. Avoid to use this option if possible. It may hide some + errors (e.g. spelling error on field name) + + Returns: + Message: The same message passed as argument. + + Raises: + ParseError: On text parsing problems. + """ + return ParseLines(text.split(b'\n' if isinstance(text, bytes) else u'\n'), + message, + allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool, + allow_unknown_field=allow_unknown_field) + + +def Merge(text, + message, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None, + allow_unknown_field=False): + """Parses a text representation of a protocol message into a message. + + Like Parse(), but allows repeated values for a non-repeated field, and uses + the last one. This means any non-repeated, top-level fields specified in text + replace those in the message. + + Args: + text (str): Message text representation. + message (Message): A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + descriptor_pool (DescriptorPool): Descriptor pool used to resolve Any types. + allow_unknown_field: if True, skip over unknown field and keep + parsing. Avoid to use this option if possible. It may hide some + errors (e.g. spelling error on field name) + + Returns: + Message: The same message passed as argument. + + Raises: + ParseError: On text parsing problems. + """ + return MergeLines( + text.split(b'\n' if isinstance(text, bytes) else u'\n'), + message, + allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool, + allow_unknown_field=allow_unknown_field) + + +def ParseLines(lines, + message, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None, + allow_unknown_field=False): + """Parses a text representation of a protocol message into a message. + + See Parse() for caveats. + + Args: + lines: An iterable of lines of a message's text representation. + message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + descriptor_pool: A DescriptorPool used to resolve Any types. + allow_unknown_field: if True, skip over unknown field and keep + parsing. Avoid to use this option if possible. It may hide some + errors (e.g. spelling error on field name) + + Returns: + The same message passed as argument. + + Raises: + ParseError: On text parsing problems. + """ + parser = _Parser(allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool, + allow_unknown_field=allow_unknown_field) + return parser.ParseLines(lines, message) + + +def MergeLines(lines, + message, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None, + allow_unknown_field=False): + """Parses a text representation of a protocol message into a message. + + See Merge() for more details. + + Args: + lines: An iterable of lines of a message's text representation. + message: A protocol buffer message to merge into. + allow_unknown_extension: if True, skip over missing extensions and keep + parsing + allow_field_number: if True, both field number and field name are allowed. + descriptor_pool: A DescriptorPool used to resolve Any types. + allow_unknown_field: if True, skip over unknown field and keep + parsing. Avoid to use this option if possible. It may hide some + errors (e.g. spelling error on field name) + + Returns: + The same message passed as argument. + + Raises: + ParseError: On text parsing problems. + """ + parser = _Parser(allow_unknown_extension, + allow_field_number, + descriptor_pool=descriptor_pool, + allow_unknown_field=allow_unknown_field) + return parser.MergeLines(lines, message) + + +class _Parser(object): + """Text format parser for protocol message.""" + + def __init__(self, + allow_unknown_extension=False, + allow_field_number=False, + descriptor_pool=None, + allow_unknown_field=False): + self.allow_unknown_extension = allow_unknown_extension + self.allow_field_number = allow_field_number + self.descriptor_pool = descriptor_pool + self.allow_unknown_field = allow_unknown_field + + def ParseLines(self, lines, message): + """Parses a text representation of a protocol message into a message.""" + self._allow_multiple_scalars = False + self._ParseOrMerge(lines, message) + return message + + def MergeLines(self, lines, message): + """Merges a text representation of a protocol message into a message.""" + self._allow_multiple_scalars = True + self._ParseOrMerge(lines, message) + return message + + def _ParseOrMerge(self, lines, message): + """Converts a text representation of a protocol message into a message. + + Args: + lines: Lines of a message's text representation. + message: A protocol buffer message to merge into. + + Raises: + ParseError: On text parsing problems. + """ + # Tokenize expects native str lines. + try: + str_lines = ( + line if isinstance(line, str) else line.decode('utf-8') + for line in lines) + tokenizer = Tokenizer(str_lines) + except UnicodeDecodeError as e: + raise ParseError from e + if message: + self.root_type = message.DESCRIPTOR.full_name + while not tokenizer.AtEnd(): + self._MergeField(tokenizer, message) + + def _MergeField(self, tokenizer, message): + """Merges a single protocol message field into a message. + + Args: + tokenizer: A tokenizer to parse the field name and values. + message: A protocol message to record the data. + + Raises: + ParseError: In case of text parsing problems. + """ + message_descriptor = message.DESCRIPTOR + if (message_descriptor.full_name == _ANY_FULL_TYPE_NAME and + tokenizer.TryConsume('[')): + type_url_prefix, packed_type_name = self._ConsumeAnyTypeUrl(tokenizer) + tokenizer.Consume(']') + tokenizer.TryConsume(':') + self._DetectSilentMarker(tokenizer, message_descriptor.full_name, + type_url_prefix + '/' + packed_type_name) + if tokenizer.TryConsume('<'): + expanded_any_end_token = '>' + else: + tokenizer.Consume('{') + expanded_any_end_token = '}' + expanded_any_sub_message = _BuildMessageFromTypeName(packed_type_name, + self.descriptor_pool) + # Direct comparison with None is used instead of implicit bool conversion + # to avoid false positives with falsy initial values, e.g. for + # google.protobuf.ListValue. + if expanded_any_sub_message is None: + raise ParseError('Type %s not found in descriptor pool' % + packed_type_name) + while not tokenizer.TryConsume(expanded_any_end_token): + if tokenizer.AtEnd(): + raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % + (expanded_any_end_token,)) + self._MergeField(tokenizer, expanded_any_sub_message) + deterministic = False + + message.Pack(expanded_any_sub_message, + type_url_prefix=type_url_prefix, + deterministic=deterministic) + return + + if tokenizer.TryConsume('['): + name = [tokenizer.ConsumeIdentifier()] + while tokenizer.TryConsume('.'): + name.append(tokenizer.ConsumeIdentifier()) + name = '.'.join(name) + + if not message_descriptor.is_extendable: + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" does not have extensions.' % + message_descriptor.full_name) + # pylint: disable=protected-access + field = message.Extensions._FindExtensionByName(name) + # pylint: enable=protected-access + if not field: + if self.allow_unknown_extension: + field = None + else: + raise tokenizer.ParseErrorPreviousToken( + 'Extension "%s" not registered. ' + 'Did you import the _pb2 module which defines it? ' + 'If you are trying to place the extension in the MessageSet ' + 'field of another message that is in an Any or MessageSet field, ' + 'that message\'s _pb2 module must be imported as well' % name) + elif message_descriptor != field.containing_type: + raise tokenizer.ParseErrorPreviousToken( + 'Extension "%s" does not extend message type "%s".' % + (name, message_descriptor.full_name)) + + tokenizer.Consume(']') + + else: + name = tokenizer.ConsumeIdentifierOrNumber() + if self.allow_field_number and name.isdigit(): + number = ParseInteger(name, True, True) + field = message_descriptor.fields_by_number.get(number, None) + if not field and message_descriptor.is_extendable: + field = message.Extensions._FindExtensionByNumber(number) + else: + field = message_descriptor.fields_by_name.get(name, None) + + # Group names are expected to be capitalized as they appear in the + # .proto file, which actually matches their type names, not their field + # names. + if not field: + field = message_descriptor.fields_by_name.get(name.lower(), None) + if field and not _IsGroupLike(field): + field = None + if field and field.message_type.name != name: + field = None + + if not field and not self.allow_unknown_field: + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" has no field named "%s".' % + (message_descriptor.full_name, name)) + + if field: + if not self._allow_multiple_scalars and field.containing_oneof: + # Check if there's a different field set in this oneof. + # Note that we ignore the case if the same field was set before, and we + # apply _allow_multiple_scalars to non-scalar fields as well. + which_oneof = message.WhichOneof(field.containing_oneof.name) + if which_oneof is not None and which_oneof != field.name: + raise tokenizer.ParseErrorPreviousToken( + 'Field "%s" is specified along with field "%s", another member ' + 'of oneof "%s" for message type "%s".' % + (field.name, which_oneof, field.containing_oneof.name, + message_descriptor.full_name)) + + if field.cpp_type == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + tokenizer.TryConsume(':') + self._DetectSilentMarker(tokenizer, message_descriptor.full_name, + field.full_name) + merger = self._MergeMessageField + else: + tokenizer.Consume(':') + self._DetectSilentMarker(tokenizer, message_descriptor.full_name, + field.full_name) + merger = self._MergeScalarField + + if (field.is_repeated and + tokenizer.TryConsume('[')): + # Short repeated format, e.g. "foo: [1, 2, 3]" + if not tokenizer.TryConsume(']'): + while True: + merger(tokenizer, message, field) + if tokenizer.TryConsume(']'): + break + tokenizer.Consume(',') + + else: + merger(tokenizer, message, field) + + else: # Proto field is unknown. + assert (self.allow_unknown_extension or self.allow_unknown_field) + self._SkipFieldContents(tokenizer, name, message_descriptor.full_name) + + # For historical reasons, fields may optionally be separated by commas or + # semicolons. + if not tokenizer.TryConsume(','): + tokenizer.TryConsume(';') + + def _LogSilentMarker(self, immediate_message_type, field_name): + pass + + def _DetectSilentMarker(self, tokenizer, immediate_message_type, field_name): + if tokenizer.contains_silent_marker_before_current_token: + self._LogSilentMarker(immediate_message_type, field_name) + + def _ConsumeAnyTypeUrl(self, tokenizer): + """Consumes a google.protobuf.Any type URL and returns the type name.""" + # Consume "type.googleapis.com/". + prefix = [tokenizer.ConsumeIdentifier()] + tokenizer.Consume('.') + prefix.append(tokenizer.ConsumeIdentifier()) + tokenizer.Consume('.') + prefix.append(tokenizer.ConsumeIdentifier()) + tokenizer.Consume('/') + # Consume the fully-qualified type name. + name = [tokenizer.ConsumeIdentifier()] + while tokenizer.TryConsume('.'): + name.append(tokenizer.ConsumeIdentifier()) + return '.'.join(prefix), '.'.join(name) + + def _MergeMessageField(self, tokenizer, message, field): + """Merges a single scalar field into a message. + + Args: + tokenizer: A tokenizer to parse the field value. + message: The message of which field is a member. + field: The descriptor of the field to be merged. + + Raises: + ParseError: In case of text parsing problems. + """ + is_map_entry = _IsMapEntry(field) + + if tokenizer.TryConsume('<'): + end_token = '>' + else: + tokenizer.Consume('{') + end_token = '}' + + if field.is_repeated: + if field.is_extension: + sub_message = message.Extensions[field].add() + elif is_map_entry: + sub_message = getattr(message, field.name).GetEntryClass()() + else: + sub_message = getattr(message, field.name).add() + else: + if field.is_extension: + if (not self._allow_multiple_scalars and + message.HasExtension(field)): + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" should not have multiple "%s" extensions.' % + (message.DESCRIPTOR.full_name, field.full_name)) + sub_message = message.Extensions[field] + else: + # Also apply _allow_multiple_scalars to message field. + # TODO: Change to _allow_singular_overwrites. + if (not self._allow_multiple_scalars and + message.HasField(field.name)): + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" should not have multiple "%s" fields.' % + (message.DESCRIPTOR.full_name, field.name)) + sub_message = getattr(message, field.name) + sub_message.SetInParent() + + while not tokenizer.TryConsume(end_token): + if tokenizer.AtEnd(): + raise tokenizer.ParseErrorPreviousToken('Expected "%s".' % (end_token,)) + self._MergeField(tokenizer, sub_message) + + if is_map_entry: + value_cpptype = field.message_type.fields_by_name['value'].cpp_type + if value_cpptype == descriptor.FieldDescriptor.CPPTYPE_MESSAGE: + value = getattr(message, field.name)[sub_message.key] + value.CopyFrom(sub_message.value) + else: + getattr(message, field.name)[sub_message.key] = sub_message.value + + def _MergeScalarField(self, tokenizer, message, field): + """Merges a single scalar field into a message. + + Args: + tokenizer: A tokenizer to parse the field value. + message: A protocol message to record the data. + field: The descriptor of the field to be merged. + + Raises: + ParseError: In case of text parsing problems. + RuntimeError: On runtime errors. + """ + _ = self.allow_unknown_extension + value = None + + if field.type in (descriptor.FieldDescriptor.TYPE_INT32, + descriptor.FieldDescriptor.TYPE_SINT32, + descriptor.FieldDescriptor.TYPE_SFIXED32): + value = _ConsumeInt32(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_INT64, + descriptor.FieldDescriptor.TYPE_SINT64, + descriptor.FieldDescriptor.TYPE_SFIXED64): + value = _ConsumeInt64(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_UINT32, + descriptor.FieldDescriptor.TYPE_FIXED32): + value = _ConsumeUint32(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_UINT64, + descriptor.FieldDescriptor.TYPE_FIXED64): + value = _ConsumeUint64(tokenizer) + elif field.type in (descriptor.FieldDescriptor.TYPE_FLOAT, + descriptor.FieldDescriptor.TYPE_DOUBLE): + value = tokenizer.ConsumeFloat() + elif field.type == descriptor.FieldDescriptor.TYPE_BOOL: + value = tokenizer.ConsumeBool() + elif field.type == descriptor.FieldDescriptor.TYPE_STRING: + value = tokenizer.ConsumeString() + elif field.type == descriptor.FieldDescriptor.TYPE_BYTES: + value = tokenizer.ConsumeByteString() + elif field.type == descriptor.FieldDescriptor.TYPE_ENUM: + value = tokenizer.ConsumeEnum(field) + else: + raise RuntimeError('Unknown field type %d' % field.type) + + if field.is_repeated: + if field.is_extension: + message.Extensions[field].append(value) + else: + getattr(message, field.name).append(value) + else: + if field.is_extension: + if (not self._allow_multiple_scalars and + field.has_presence and + message.HasExtension(field)): + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" should not have multiple "%s" extensions.' % + (message.DESCRIPTOR.full_name, field.full_name)) + else: + message.Extensions[field] = value + else: + duplicate_error = False + if not self._allow_multiple_scalars: + if field.has_presence: + duplicate_error = message.HasField(field.name) + else: + # For field that doesn't represent presence, try best effort to + # check multiple scalars by compare to default values. + duplicate_error = not decoder.IsDefaultScalarValue( + getattr(message, field.name) + ) + + if duplicate_error: + raise tokenizer.ParseErrorPreviousToken( + 'Message type "%s" should not have multiple "%s" fields.' % + (message.DESCRIPTOR.full_name, field.name)) + else: + setattr(message, field.name, value) + + def _SkipFieldContents(self, tokenizer, field_name, immediate_message_type): + """Skips over contents (value or message) of a field. + + Args: + tokenizer: A tokenizer to parse the field name and values. + field_name: The field name currently being parsed. + immediate_message_type: The type of the message immediately containing + the silent marker. + """ + # Try to guess the type of this field. + # If this field is not a message, there should be a ":" between the + # field name and the field value and also the field value should not + # start with "{" or "<" which indicates the beginning of a message body. + # If there is no ":" or there is a "{" or "<" after ":", this field has + # to be a message or the input is ill-formed. + if tokenizer.TryConsume( + ':') and not tokenizer.LookingAt('{') and not tokenizer.LookingAt('<'): + self._DetectSilentMarker(tokenizer, immediate_message_type, field_name) + if tokenizer.LookingAt('['): + self._SkipRepeatedFieldValue(tokenizer, immediate_message_type) + else: + self._SkipFieldValue(tokenizer) + else: + self._DetectSilentMarker(tokenizer, immediate_message_type, field_name) + self._SkipFieldMessage(tokenizer, immediate_message_type) + + def _SkipField(self, tokenizer, immediate_message_type): + """Skips over a complete field (name and value/message). + + Args: + tokenizer: A tokenizer to parse the field name and values. + immediate_message_type: The type of the message immediately containing + the silent marker. + """ + field_name = '' + if tokenizer.TryConsume('['): + # Consume extension or google.protobuf.Any type URL + field_name += '[' + tokenizer.ConsumeIdentifier() + num_identifiers = 1 + while tokenizer.TryConsume('.'): + field_name += '.' + tokenizer.ConsumeIdentifier() + num_identifiers += 1 + # This is possibly a type URL for an Any message. + if num_identifiers == 3 and tokenizer.TryConsume('/'): + field_name += '/' + tokenizer.ConsumeIdentifier() + while tokenizer.TryConsume('.'): + field_name += '.' + tokenizer.ConsumeIdentifier() + tokenizer.Consume(']') + field_name += ']' + else: + field_name += tokenizer.ConsumeIdentifierOrNumber() + + self._SkipFieldContents(tokenizer, field_name, immediate_message_type) + + # For historical reasons, fields may optionally be separated by commas or + # semicolons. + if not tokenizer.TryConsume(','): + tokenizer.TryConsume(';') + + def _SkipFieldMessage(self, tokenizer, immediate_message_type): + """Skips over a field message. + + Args: + tokenizer: A tokenizer to parse the field name and values. + immediate_message_type: The type of the message immediately containing + the silent marker + """ + if tokenizer.TryConsume('<'): + delimiter = '>' + else: + tokenizer.Consume('{') + delimiter = '}' + + while not tokenizer.LookingAt('>') and not tokenizer.LookingAt('}'): + self._SkipField(tokenizer, immediate_message_type) + + tokenizer.Consume(delimiter) + + def _SkipFieldValue(self, tokenizer): + """Skips over a field value. + + Args: + tokenizer: A tokenizer to parse the field name and values. + + Raises: + ParseError: In case an invalid field value is found. + """ + if (not tokenizer.TryConsumeByteString()and + not tokenizer.TryConsumeIdentifier() and + not _TryConsumeInt64(tokenizer) and + not _TryConsumeUint64(tokenizer) and + not tokenizer.TryConsumeFloat()): + raise ParseError('Invalid field value: ' + tokenizer.token) + + def _SkipRepeatedFieldValue(self, tokenizer, immediate_message_type): + """Skips over a repeated field value. + + Args: + tokenizer: A tokenizer to parse the field value. + """ + tokenizer.Consume('[') + if not tokenizer.TryConsume(']'): + while True: + if tokenizer.LookingAt('<') or tokenizer.LookingAt('{'): + self._SkipFieldMessage(tokenizer, immediate_message_type) + else: + self._SkipFieldValue(tokenizer) + if tokenizer.TryConsume(']'): + break + tokenizer.Consume(',') + + +class Tokenizer(object): + """Protocol buffer text representation tokenizer. + + This class handles the lower level string parsing by splitting it into + meaningful tokens. + + It was directly ported from the Java protocol buffer API. + """ + + _WHITESPACE = re.compile(r'\s+') + _COMMENT = re.compile(r'(\s*#.*$)', re.MULTILINE) + _WHITESPACE_OR_COMMENT = re.compile(r'(\s|(#.*$))+', re.MULTILINE) + _TOKEN = re.compile('|'.join([ + r'[a-zA-Z_][0-9a-zA-Z_+-]*', # an identifier + r'([0-9+-]|(\.[0-9]))[0-9a-zA-Z_.+-]*', # a number + ] + [ # quoted str for each quote mark + # Avoid backtracking! https://stackoverflow.com/a/844267 + r'{qt}[^{qt}\n\\]*((\\.)+[^{qt}\n\\]*)*({qt}|\\?$)'.format(qt=mark) + for mark in _QUOTES + ])) + + _IDENTIFIER = re.compile(r'[^\d\W]\w*') + _IDENTIFIER_OR_NUMBER = re.compile(r'\w+') + + def __init__(self, lines, skip_comments=True): + self._position = 0 + self._line = -1 + self._column = 0 + self._token_start = None + self.token = '' + self._lines = iter(lines) + self._current_line = '' + self._previous_line = 0 + self._previous_column = 0 + self._more_lines = True + self._skip_comments = skip_comments + self._whitespace_pattern = (skip_comments and self._WHITESPACE_OR_COMMENT + or self._WHITESPACE) + self.contains_silent_marker_before_current_token = False + + self._SkipWhitespace() + self.NextToken() + + def LookingAt(self, token): + return self.token == token + + def AtEnd(self): + """Checks the end of the text was reached. + + Returns: + True iff the end was reached. + """ + return not self.token + + def _PopLine(self): + while len(self._current_line) <= self._column: + try: + self._current_line = next(self._lines) + except StopIteration: + self._current_line = '' + self._more_lines = False + return + else: + self._line += 1 + self._column = 0 + + def _SkipWhitespace(self): + while True: + self._PopLine() + match = self._whitespace_pattern.match(self._current_line, self._column) + if not match: + break + self.contains_silent_marker_before_current_token = match.group(0) == ( + ' ' + _DEBUG_STRING_SILENT_MARKER) + length = len(match.group(0)) + self._column += length + + def TryConsume(self, token): + """Tries to consume a given piece of text. + + Args: + token: Text to consume. + + Returns: + True iff the text was consumed. + """ + if self.token == token: + self.NextToken() + return True + return False + + def Consume(self, token): + """Consumes a piece of text. + + Args: + token: Text to consume. + + Raises: + ParseError: If the text couldn't be consumed. + """ + if not self.TryConsume(token): + raise self.ParseError('Expected "%s".' % token) + + def ConsumeComment(self): + result = self.token + if not self._COMMENT.match(result): + raise self.ParseError('Expected comment.') + self.NextToken() + return result + + def ConsumeCommentOrTrailingComment(self): + """Consumes a comment, returns a 2-tuple (trailing bool, comment str).""" + + # Tokenizer initializes _previous_line and _previous_column to 0. As the + # tokenizer starts, it looks like there is a previous token on the line. + just_started = self._line == 0 and self._column == 0 + + before_parsing = self._previous_line + comment = self.ConsumeComment() + + # A trailing comment is a comment on the same line than the previous token. + trailing = (self._previous_line == before_parsing + and not just_started) + + return trailing, comment + + def TryConsumeIdentifier(self): + try: + self.ConsumeIdentifier() + return True + except ParseError: + return False + + def ConsumeIdentifier(self): + """Consumes protocol message field identifier. + + Returns: + Identifier string. + + Raises: + ParseError: If an identifier couldn't be consumed. + """ + result = self.token + if not self._IDENTIFIER.match(result): + raise self.ParseError('Expected identifier.') + self.NextToken() + return result + + def TryConsumeIdentifierOrNumber(self): + try: + self.ConsumeIdentifierOrNumber() + return True + except ParseError: + return False + + def ConsumeIdentifierOrNumber(self): + """Consumes protocol message field identifier. + + Returns: + Identifier string. + + Raises: + ParseError: If an identifier couldn't be consumed. + """ + result = self.token + if not self._IDENTIFIER_OR_NUMBER.match(result): + raise self.ParseError('Expected identifier or number, got %s.' % result) + self.NextToken() + return result + + def TryConsumeInteger(self): + try: + self.ConsumeInteger() + return True + except ParseError: + return False + + def ConsumeInteger(self): + """Consumes an integer number. + + Returns: + The integer parsed. + + Raises: + ParseError: If an integer couldn't be consumed. + """ + try: + result = _ParseAbstractInteger(self.token) + except ValueError as e: + raise self.ParseError(str(e)) + self.NextToken() + return result + + def TryConsumeFloat(self): + try: + self.ConsumeFloat() + return True + except ParseError: + return False + + def ConsumeFloat(self): + """Consumes an floating point number. + + Returns: + The number parsed. + + Raises: + ParseError: If a floating point number couldn't be consumed. + """ + try: + result = ParseFloat(self.token) + except ValueError as e: + raise self.ParseError(str(e)) + self.NextToken() + return result + + def ConsumeBool(self): + """Consumes a boolean value. + + Returns: + The bool parsed. + + Raises: + ParseError: If a boolean value couldn't be consumed. + """ + try: + result = ParseBool(self.token) + except ValueError as e: + raise self.ParseError(str(e)) + self.NextToken() + return result + + def TryConsumeByteString(self): + try: + self.ConsumeByteString() + return True + except ParseError: + return False + + def ConsumeString(self): + """Consumes a string value. + + Returns: + The string parsed. + + Raises: + ParseError: If a string value couldn't be consumed. + """ + the_bytes = self.ConsumeByteString() + try: + return str(the_bytes, 'utf-8') + except UnicodeDecodeError as e: + raise self._StringParseError(e) + + def ConsumeByteString(self): + """Consumes a byte array value. + + Returns: + The array parsed (as a string). + + Raises: + ParseError: If a byte array value couldn't be consumed. + """ + the_list = [self._ConsumeSingleByteString()] + while self.token and self.token[0] in _QUOTES: + the_list.append(self._ConsumeSingleByteString()) + return b''.join(the_list) + + def _ConsumeSingleByteString(self): + """Consume one token of a string literal. + + String literals (whether bytes or text) can come in multiple adjacent + tokens which are automatically concatenated, like in C or Python. This + method only consumes one token. + + Returns: + The token parsed. + Raises: + ParseError: When the wrong format data is found. + """ + text = self.token + if len(text) < 1 or text[0] not in _QUOTES: + raise self.ParseError('Expected string but found: %r' % (text,)) + + if len(text) < 2 or text[-1] != text[0]: + raise self.ParseError('String missing ending quote: %r' % (text,)) + + try: + result = text_encoding.CUnescape(text[1:-1]) + except ValueError as e: + raise self.ParseError(str(e)) + self.NextToken() + return result + + def ConsumeEnum(self, field): + try: + result = ParseEnum(field, self.token) + except ValueError as e: + raise self.ParseError(str(e)) + self.NextToken() + return result + + def ParseErrorPreviousToken(self, message): + """Creates and *returns* a ParseError for the previously read token. + + Args: + message: A message to set for the exception. + + Returns: + A ParseError instance. + """ + return ParseError(message, self._previous_line + 1, + self._previous_column + 1) + + def ParseError(self, message): + """Creates and *returns* a ParseError for the current token.""" + return ParseError('\'' + self._current_line + '\': ' + message, + self._line + 1, self._column + 1) + + def _StringParseError(self, e): + return self.ParseError('Couldn\'t parse string: ' + str(e)) + + def NextToken(self): + """Reads the next meaningful token.""" + self._previous_line = self._line + self._previous_column = self._column + self.contains_silent_marker_before_current_token = False + + self._column += len(self.token) + self._SkipWhitespace() + + if not self._more_lines: + self.token = '' + return + + match = self._TOKEN.match(self._current_line, self._column) + if not match and not self._skip_comments: + match = self._COMMENT.match(self._current_line, self._column) + if match: + token = match.group(0) + self.token = token + else: + self.token = self._current_line[self._column] + +# Aliased so it can still be accessed by current visibility violators. +# TODO: Migrate violators to textformat_tokenizer. +_Tokenizer = Tokenizer # pylint: disable=invalid-name + + +def _ConsumeInt32(tokenizer): + """Consumes a signed 32bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. + + Raises: + ParseError: If a signed 32bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=True, is_long=False) + + +def _ConsumeUint32(tokenizer): + """Consumes an unsigned 32bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. + + Raises: + ParseError: If an unsigned 32bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=False, is_long=False) + + +def _TryConsumeInt64(tokenizer): + try: + _ConsumeInt64(tokenizer) + return True + except ParseError: + return False + + +def _ConsumeInt64(tokenizer): + """Consumes a signed 32bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. + + Raises: + ParseError: If a signed 32bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=True, is_long=True) + + +def _TryConsumeUint64(tokenizer): + try: + _ConsumeUint64(tokenizer) + return True + except ParseError: + return False + + +def _ConsumeUint64(tokenizer): + """Consumes an unsigned 64bit integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + + Returns: + The integer parsed. + + Raises: + ParseError: If an unsigned 64bit integer couldn't be consumed. + """ + return _ConsumeInteger(tokenizer, is_signed=False, is_long=True) + + +def _ConsumeInteger(tokenizer, is_signed=False, is_long=False): + """Consumes an integer number from tokenizer. + + Args: + tokenizer: A tokenizer used to parse the number. + is_signed: True if a signed integer must be parsed. + is_long: True if a long integer must be parsed. + + Returns: + The integer parsed. + + Raises: + ParseError: If an integer with given characteristics couldn't be consumed. + """ + try: + result = ParseInteger(tokenizer.token, is_signed=is_signed, is_long=is_long) + except ValueError as e: + raise tokenizer.ParseError(str(e)) + tokenizer.NextToken() + return result + + +def ParseInteger(text, is_signed=False, is_long=False): + """Parses an integer. + + Args: + text: The text to parse. + is_signed: True if a signed integer must be parsed. + is_long: True if a long integer must be parsed. + + Returns: + The integer value. + + Raises: + ValueError: Thrown Iff the text is not a valid integer. + """ + # Do the actual parsing. Exception handling is propagated to caller. + result = _ParseAbstractInteger(text) + + # Check if the integer is sane. Exceptions handled by callers. + checker = _INTEGER_CHECKERS[2 * int(is_long) + int(is_signed)] + checker.CheckValue(result) + return result + + +def _ParseAbstractInteger(text): + """Parses an integer without checking size/signedness. + + Args: + text: The text to parse. + + Returns: + The integer value. + + Raises: + ValueError: Thrown Iff the text is not a valid integer. + """ + # Do the actual parsing. Exception handling is propagated to caller. + orig_text = text + c_octal_match = re.match(r'(-?)0(\d+)$', text) + if c_octal_match: + # Python 3 no longer supports 0755 octal syntax without the 'o', so + # we always use the '0o' prefix for multi-digit numbers starting with 0. + text = c_octal_match.group(1) + '0o' + c_octal_match.group(2) + try: + return int(text, 0) + except ValueError: + raise ValueError('Couldn\'t parse integer: %s' % orig_text) + + +def ParseFloat(text): + """Parse a floating point number. + + Args: + text: Text to parse. + + Returns: + The number parsed. + + Raises: + ValueError: If a floating point number couldn't be parsed. + """ + if _FLOAT_OCTAL_PREFIX.match(text): + raise ValueError('Invalid octal float: %s' % text) + try: + # Assume Python compatible syntax. + return float(text) + except ValueError: + # Check alternative spellings. + if _FLOAT_INFINITY.match(text): + if text[0] == '-': + return float('-inf') + else: + return float('inf') + elif _FLOAT_NAN.match(text): + return float('nan') + else: + # assume '1.0f' format + try: + return float(text.rstrip('fF')) + except ValueError: + raise ValueError("Couldn't parse float: %s" % text) + + +def ParseBool(text): + """Parse a boolean value. + + Args: + text: Text to parse. + + Returns: + Boolean values parsed + + Raises: + ValueError: If text is not a valid boolean. + """ + if text in ('true', 't', '1', 'True'): + return True + elif text in ('false', 'f', '0', 'False'): + return False + else: + raise ValueError('Expected "true" or "false".') + + +def ParseEnum(field, value): + """Parse an enum value. + + The value can be specified by a number (the enum value), or by + a string literal (the enum name). + + Args: + field: Enum field descriptor. + value: String value. + + Returns: + Enum value number. + + Raises: + ValueError: If the enum value could not be parsed. + """ + enum_descriptor = field.enum_type + try: + number = int(value, 0) + except ValueError: + # Identifier. + enum_value = enum_descriptor.values_by_name.get(value, None) + if enum_value is None: + raise ValueError('Enum type "%s" has no value named %s.' % + (enum_descriptor.full_name, value)) + else: + if not field.enum_type.is_closed: + return number + enum_value = enum_descriptor.values_by_number.get(number, None) + if enum_value is None: + raise ValueError('Enum type "%s" has no value with number %d.' % + (enum_descriptor.full_name, number)) + return enum_value.number diff --git a/lib/python3.12/site-packages/google/protobuf/timestamp.py b/lib/python3.12/site-packages/google/protobuf/timestamp.py new file mode 100644 index 0000000000000000000000000000000000000000..b23b9f11af02984469f565ae253730c1fdda0c1d --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/timestamp.py @@ -0,0 +1,112 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains the Timestamp helper APIs.""" + +import datetime +from typing import Optional + +from google.protobuf.timestamp_pb2 import Timestamp + + +def from_json_string(value: str) -> Timestamp: + """Parse a RFC 3339 date string format to Timestamp. + + Args: + value: A date string. Any fractional digits (or none) and any offset are + accepted as long as they fit into nano-seconds precision. Example of + accepted format: '1972-01-01T10:00:20.021-05:00' + + Raises: + ValueError: On parsing problems. + """ + timestamp = Timestamp() + timestamp.FromJsonString(value) + return timestamp + + +def from_microseconds(micros: float) -> Timestamp: + """Converts microseconds since epoch to Timestamp.""" + timestamp = Timestamp() + timestamp.FromMicroseconds(micros) + return timestamp + + +def from_milliseconds(millis: float) -> Timestamp: + """Converts milliseconds since epoch to Timestamp.""" + timestamp = Timestamp() + timestamp.FromMilliseconds(millis) + return timestamp + + +def from_nanoseconds(nanos: float) -> Timestamp: + """Converts nanoseconds since epoch to Timestamp.""" + timestamp = Timestamp() + timestamp.FromNanoseconds(nanos) + return timestamp + + +def from_seconds(seconds: float) -> Timestamp: + """Converts seconds since epoch to Timestamp.""" + timestamp = Timestamp() + timestamp.FromSeconds(seconds) + return timestamp + + +def from_current_time() -> Timestamp: + """Converts the current UTC to Timestamp.""" + timestamp = Timestamp() + timestamp.FromDatetime(datetime.datetime.now(tz=datetime.timezone.utc)) + return timestamp + + +def to_json_string(ts: Timestamp) -> str: + """Converts Timestamp to RFC 3339 date string format. + + Returns: + A string converted from timestamp. The string is always Z-normalized + and uses 3, 6 or 9 fractional digits as required to represent the + exact time. Example of the return format: '1972-01-01T10:00:20.021Z' + """ + return ts.ToJsonString() + + +def to_microseconds(ts: Timestamp) -> int: + """Converts Timestamp to microseconds since epoch.""" + return ts.ToMicroseconds() + + +def to_milliseconds(ts: Timestamp) -> int: + """Converts Timestamp to milliseconds since epoch.""" + return ts.ToMilliseconds() + + +def to_nanoseconds(ts: Timestamp) -> int: + """Converts Timestamp to nanoseconds since epoch.""" + return ts.ToNanoseconds() + + +def to_seconds(ts: Timestamp) -> int: + """Converts Timestamp to seconds since epoch.""" + return ts.ToSeconds() + + +def to_datetime( + ts: Timestamp, tz: Optional[datetime.tzinfo] = None +) -> datetime.datetime: + """Converts Timestamp to a datetime. + + Args: + tz: A datetime.tzinfo subclass; defaults to None. + + Returns: + If tzinfo is None, returns a timezone-naive UTC datetime (with no timezone + information, i.e. not aware that it's UTC). + + Otherwise, returns a timezone-aware datetime in the input timezone. + """ + return ts.ToDatetime(tzinfo=tz) diff --git a/lib/python3.12/site-packages/google/protobuf/timestamp_pb2.py b/lib/python3.12/site-packages/google/protobuf/timestamp_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..1c740a03b0525e8df5ca8edfe5c8ddd3a5536133 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/timestamp_pb2.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/timestamp.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/timestamp.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1fgoogle/protobuf/timestamp.proto\x12\x0fgoogle.protobuf\";\n\tTimestamp\x12\x18\n\x07seconds\x18\x01 \x01(\x03R\x07seconds\x12\x14\n\x05nanos\x18\x02 \x01(\x05R\x05nanosB\x85\x01\n\x13\x63om.google.protobufB\x0eTimestampProtoP\x01Z2google.golang.org/protobuf/types/known/timestamppb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.timestamp_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\016TimestampProtoP\001Z2google.golang.org/protobuf/types/known/timestamppb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_TIMESTAMP']._serialized_start=52 + _globals['_TIMESTAMP']._serialized_end=111 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/type_pb2.py b/lib/python3.12/site-packages/google/protobuf/type_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..0bad460d0e26e1beda7d37bea8d5920346266627 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/type_pb2.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/type.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/type.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from google.protobuf import any_pb2 as google_dot_protobuf_dot_any__pb2 +from google.protobuf import source_context_pb2 as google_dot_protobuf_dot_source__context__pb2 + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1agoogle/protobuf/type.proto\x12\x0fgoogle.protobuf\x1a\x19google/protobuf/any.proto\x1a$google/protobuf/source_context.proto\"\xa7\x02\n\x04Type\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12.\n\x06\x66ields\x18\x02 \x03(\x0b\x32\x16.google.protobuf.FieldR\x06\x66ields\x12\x16\n\x06oneofs\x18\x03 \x03(\tR\x06oneofs\x12\x31\n\x07options\x18\x04 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x45\n\x0esource_context\x18\x05 \x01(\x0b\x32\x1e.google.protobuf.SourceContextR\rsourceContext\x12/\n\x06syntax\x18\x06 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\x12\x18\n\x07\x65\x64ition\x18\x07 \x01(\tR\x07\x65\x64ition\"\xb4\x06\n\x05\x46ield\x12/\n\x04kind\x18\x01 \x01(\x0e\x32\x1b.google.protobuf.Field.KindR\x04kind\x12\x44\n\x0b\x63\x61rdinality\x18\x02 \x01(\x0e\x32\".google.protobuf.Field.CardinalityR\x0b\x63\x61rdinality\x12\x16\n\x06number\x18\x03 \x01(\x05R\x06number\x12\x12\n\x04name\x18\x04 \x01(\tR\x04name\x12\x19\n\x08type_url\x18\x06 \x01(\tR\x07typeUrl\x12\x1f\n\x0boneof_index\x18\x07 \x01(\x05R\noneofIndex\x12\x16\n\x06packed\x18\x08 \x01(\x08R\x06packed\x12\x31\n\x07options\x18\t \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x1b\n\tjson_name\x18\n \x01(\tR\x08jsonName\x12#\n\rdefault_value\x18\x0b \x01(\tR\x0c\x64\x65\x66\x61ultValue\"\xc8\x02\n\x04Kind\x12\x10\n\x0cTYPE_UNKNOWN\x10\x00\x12\x0f\n\x0bTYPE_DOUBLE\x10\x01\x12\x0e\n\nTYPE_FLOAT\x10\x02\x12\x0e\n\nTYPE_INT64\x10\x03\x12\x0f\n\x0bTYPE_UINT64\x10\x04\x12\x0e\n\nTYPE_INT32\x10\x05\x12\x10\n\x0cTYPE_FIXED64\x10\x06\x12\x10\n\x0cTYPE_FIXED32\x10\x07\x12\r\n\tTYPE_BOOL\x10\x08\x12\x0f\n\x0bTYPE_STRING\x10\t\x12\x0e\n\nTYPE_GROUP\x10\n\x12\x10\n\x0cTYPE_MESSAGE\x10\x0b\x12\x0e\n\nTYPE_BYTES\x10\x0c\x12\x0f\n\x0bTYPE_UINT32\x10\r\x12\r\n\tTYPE_ENUM\x10\x0e\x12\x11\n\rTYPE_SFIXED32\x10\x0f\x12\x11\n\rTYPE_SFIXED64\x10\x10\x12\x0f\n\x0bTYPE_SINT32\x10\x11\x12\x0f\n\x0bTYPE_SINT64\x10\x12\"t\n\x0b\x43\x61rdinality\x12\x17\n\x13\x43\x41RDINALITY_UNKNOWN\x10\x00\x12\x18\n\x14\x43\x41RDINALITY_OPTIONAL\x10\x01\x12\x18\n\x14\x43\x41RDINALITY_REQUIRED\x10\x02\x12\x18\n\x14\x43\x41RDINALITY_REPEATED\x10\x03\"\x99\x02\n\x04\x45num\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x38\n\tenumvalue\x18\x02 \x03(\x0b\x32\x1a.google.protobuf.EnumValueR\tenumvalue\x12\x31\n\x07options\x18\x03 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\x12\x45\n\x0esource_context\x18\x04 \x01(\x0b\x32\x1e.google.protobuf.SourceContextR\rsourceContext\x12/\n\x06syntax\x18\x05 \x01(\x0e\x32\x17.google.protobuf.SyntaxR\x06syntax\x12\x18\n\x07\x65\x64ition\x18\x06 \x01(\tR\x07\x65\x64ition\"j\n\tEnumValue\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12\x16\n\x06number\x18\x02 \x01(\x05R\x06number\x12\x31\n\x07options\x18\x03 \x03(\x0b\x32\x17.google.protobuf.OptionR\x07options\"H\n\x06Option\x12\x12\n\x04name\x18\x01 \x01(\tR\x04name\x12*\n\x05value\x18\x02 \x01(\x0b\x32\x14.google.protobuf.AnyR\x05value*C\n\x06Syntax\x12\x11\n\rSYNTAX_PROTO2\x10\x00\x12\x11\n\rSYNTAX_PROTO3\x10\x01\x12\x13\n\x0fSYNTAX_EDITIONS\x10\x02\x42{\n\x13\x63om.google.protobufB\tTypeProtoP\x01Z-google.golang.org/protobuf/types/known/typepb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.type_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\tTypeProtoP\001Z-google.golang.org/protobuf/types/known/typepb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_SYNTAX']._serialized_start=1699 + _globals['_SYNTAX']._serialized_end=1766 + _globals['_TYPE']._serialized_start=113 + _globals['_TYPE']._serialized_end=408 + _globals['_FIELD']._serialized_start=411 + _globals['_FIELD']._serialized_end=1231 + _globals['_FIELD_KIND']._serialized_start=785 + _globals['_FIELD_KIND']._serialized_end=1113 + _globals['_FIELD_CARDINALITY']._serialized_start=1115 + _globals['_FIELD_CARDINALITY']._serialized_end=1231 + _globals['_ENUM']._serialized_start=1234 + _globals['_ENUM']._serialized_end=1515 + _globals['_ENUMVALUE']._serialized_start=1517 + _globals['_ENUMVALUE']._serialized_end=1623 + _globals['_OPTION']._serialized_start=1625 + _globals['_OPTION']._serialized_end=1697 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/google/protobuf/unknown_fields.py b/lib/python3.12/site-packages/google/protobuf/unknown_fields.py new file mode 100644 index 0000000000000000000000000000000000000000..6c16a2d30ced679544f8134812e99118caed8348 --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/unknown_fields.py @@ -0,0 +1,96 @@ +# Protocol Buffers - Google's data interchange format +# Copyright 2008 Google Inc. All rights reserved. +# +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file or at +# https://developers.google.com/open-source/licenses/bsd + +"""Contains Unknown Fields APIs. + +Simple usage example: + unknown_field_set = UnknownFieldSet(message) + for unknown_field in unknown_field_set: + wire_type = unknown_field.wire_type + field_number = unknown_field.field_number + data = unknown_field.data +""" + + +from google.protobuf.internal import api_implementation + +if api_implementation._c_module is not None: # pylint: disable=protected-access + UnknownFieldSet = api_implementation._c_module.UnknownFieldSet # pylint: disable=protected-access +else: + from google.protobuf.internal import decoder # pylint: disable=g-import-not-at-top + from google.protobuf.internal import wire_format # pylint: disable=g-import-not-at-top + + class UnknownField: + """A parsed unknown field.""" + + # Disallows assignment to other attributes. + __slots__ = ['_field_number', '_wire_type', '_data'] + + def __init__(self, field_number, wire_type, data): + self._field_number = field_number + self._wire_type = wire_type + self._data = data + return + + @property + def field_number(self): + return self._field_number + + @property + def wire_type(self): + return self._wire_type + + @property + def data(self): + return self._data + + class UnknownFieldSet: + """UnknownField container.""" + + # Disallows assignment to other attributes. + __slots__ = ['_values'] + + def __init__(self, msg): + + def InternalAdd(field_number, wire_type, data): + unknown_field = UnknownField(field_number, wire_type, data) + self._values.append(unknown_field) + + self._values = [] + msg_des = msg.DESCRIPTOR + # pylint: disable=protected-access + unknown_fields = msg._unknown_fields + if (msg_des.has_options and + msg_des.GetOptions().message_set_wire_format): + local_decoder = decoder.UnknownMessageSetItemDecoder() + for _, buffer in unknown_fields: + (field_number, data) = local_decoder(memoryview(buffer)) + InternalAdd(field_number, wire_format.WIRETYPE_LENGTH_DELIMITED, data) + else: + for tag_bytes, buffer in unknown_fields: + field_number, wire_type = decoder.DecodeTag(tag_bytes) + if field_number == 0: + raise RuntimeError('Field number 0 is illegal.') + (data, _) = decoder._DecodeUnknownField( + memoryview(buffer), 0, len(buffer), field_number, wire_type + ) + InternalAdd(field_number, wire_type, data) + + def __getitem__(self, index): + size = len(self._values) + if index < 0: + index += size + if index < 0 or index >= size: + raise IndexError('index %d out of range'.index) + + return self._values[index] + + def __len__(self): + return len(self._values) + + def __iter__(self): + return iter(self._values) diff --git a/lib/python3.12/site-packages/google/protobuf/util/__init__.py b/lib/python3.12/site-packages/google/protobuf/util/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/google/protobuf/util/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/google/protobuf/util/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..14300e0c42d30eeae848f541b85470be45a030ae Binary files /dev/null and b/lib/python3.12/site-packages/google/protobuf/util/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/google/protobuf/wrappers_pb2.py b/lib/python3.12/site-packages/google/protobuf/wrappers_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..12882e3218bc2e974a2799735dd5b789dade5c0f --- /dev/null +++ b/lib/python3.12/site-packages/google/protobuf/wrappers_pb2.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# NO CHECKED-IN PROTOBUF GENCODE +# source: google/protobuf/wrappers.proto +# Protobuf Python Version: 6.33.5 +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import descriptor_pool as _descriptor_pool +from google.protobuf import runtime_version as _runtime_version +from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder +_runtime_version.ValidateProtobufRuntimeVersion( + _runtime_version.Domain.PUBLIC, + 6, + 33, + 5, + '', + 'google/protobuf/wrappers.proto' +) +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1egoogle/protobuf/wrappers.proto\x12\x0fgoogle.protobuf\"#\n\x0b\x44oubleValue\x12\x14\n\x05value\x18\x01 \x01(\x01R\x05value\"\"\n\nFloatValue\x12\x14\n\x05value\x18\x01 \x01(\x02R\x05value\"\"\n\nInt64Value\x12\x14\n\x05value\x18\x01 \x01(\x03R\x05value\"#\n\x0bUInt64Value\x12\x14\n\x05value\x18\x01 \x01(\x04R\x05value\"\"\n\nInt32Value\x12\x14\n\x05value\x18\x01 \x01(\x05R\x05value\"#\n\x0bUInt32Value\x12\x14\n\x05value\x18\x01 \x01(\rR\x05value\"!\n\tBoolValue\x12\x14\n\x05value\x18\x01 \x01(\x08R\x05value\"#\n\x0bStringValue\x12\x14\n\x05value\x18\x01 \x01(\tR\x05value\"\"\n\nBytesValue\x12\x14\n\x05value\x18\x01 \x01(\x0cR\x05valueB\x83\x01\n\x13\x63om.google.protobufB\rWrappersProtoP\x01Z1google.golang.org/protobuf/types/known/wrapperspb\xf8\x01\x01\xa2\x02\x03GPB\xaa\x02\x1eGoogle.Protobuf.WellKnownTypesb\x06proto3') + +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'google.protobuf.wrappers_pb2', _globals) +if not _descriptor._USE_C_DESCRIPTORS: + _globals['DESCRIPTOR']._loaded_options = None + _globals['DESCRIPTOR']._serialized_options = b'\n\023com.google.protobufB\rWrappersProtoP\001Z1google.golang.org/protobuf/types/known/wrapperspb\370\001\001\242\002\003GPB\252\002\036Google.Protobuf.WellKnownTypes' + _globals['_DOUBLEVALUE']._serialized_start=51 + _globals['_DOUBLEVALUE']._serialized_end=86 + _globals['_FLOATVALUE']._serialized_start=88 + _globals['_FLOATVALUE']._serialized_end=122 + _globals['_INT64VALUE']._serialized_start=124 + _globals['_INT64VALUE']._serialized_end=158 + _globals['_UINT64VALUE']._serialized_start=160 + _globals['_UINT64VALUE']._serialized_end=195 + _globals['_INT32VALUE']._serialized_start=197 + _globals['_INT32VALUE']._serialized_end=231 + _globals['_UINT32VALUE']._serialized_start=233 + _globals['_UINT32VALUE']._serialized_end=268 + _globals['_BOOLVALUE']._serialized_start=270 + _globals['_BOOLVALUE']._serialized_end=303 + _globals['_STRINGVALUE']._serialized_start=305 + _globals['_STRINGVALUE']._serialized_end=340 + _globals['_BYTESVALUE']._serialized_start=342 + _globals['_BYTESVALUE']._serialized_end=376 +# @@protoc_insertion_point(module_scope) diff --git a/lib/python3.12/site-packages/pexpect/ANSI.py b/lib/python3.12/site-packages/pexpect/ANSI.py new file mode 100644 index 0000000000000000000000000000000000000000..1cd2e90e7ab0c54430801b53d84ef9bb8d749485 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/ANSI.py @@ -0,0 +1,351 @@ +'''This implements an ANSI (VT100) terminal emulator as a subclass of screen. + +PEXPECT LICENSE + + This license is approved by the OSI and FSF as GPL-compatible. + http://opensource.org/licenses/isc-license.txt + + Copyright (c) 2012, Noah Spurrier + PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY + PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE + COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +''' + +# references: +# http://en.wikipedia.org/wiki/ANSI_escape_code +# http://www.retards.org/terminals/vt102.html +# http://vt100.net/docs/vt102-ug/contents.html +# http://vt100.net/docs/vt220-rm/ +# http://www.termsys.demon.co.uk/vtansi.htm + +from . import screen +from . import FSM +import string + +# +# The 'Do.*' functions are helper functions for the ANSI class. +# +def DoEmit (fsm): + + screen = fsm.memory[0] + screen.write_ch(fsm.input_symbol) + +def DoStartNumber (fsm): + + fsm.memory.append (fsm.input_symbol) + +def DoBuildNumber (fsm): + + ns = fsm.memory.pop() + ns = ns + fsm.input_symbol + fsm.memory.append (ns) + +def DoBackOne (fsm): + + screen = fsm.memory[0] + screen.cursor_back () + +def DoBack (fsm): + + count = int(fsm.memory.pop()) + screen = fsm.memory[0] + screen.cursor_back (count) + +def DoDownOne (fsm): + + screen = fsm.memory[0] + screen.cursor_down () + +def DoDown (fsm): + + count = int(fsm.memory.pop()) + screen = fsm.memory[0] + screen.cursor_down (count) + +def DoForwardOne (fsm): + + screen = fsm.memory[0] + screen.cursor_forward () + +def DoForward (fsm): + + count = int(fsm.memory.pop()) + screen = fsm.memory[0] + screen.cursor_forward (count) + +def DoUpReverse (fsm): + + screen = fsm.memory[0] + screen.cursor_up_reverse() + +def DoUpOne (fsm): + + screen = fsm.memory[0] + screen.cursor_up () + +def DoUp (fsm): + + count = int(fsm.memory.pop()) + screen = fsm.memory[0] + screen.cursor_up (count) + +def DoHome (fsm): + + c = int(fsm.memory.pop()) + r = int(fsm.memory.pop()) + screen = fsm.memory[0] + screen.cursor_home (r,c) + +def DoHomeOrigin (fsm): + + c = 1 + r = 1 + screen = fsm.memory[0] + screen.cursor_home (r,c) + +def DoEraseDown (fsm): + + screen = fsm.memory[0] + screen.erase_down() + +def DoErase (fsm): + + arg = int(fsm.memory.pop()) + screen = fsm.memory[0] + if arg == 0: + screen.erase_down() + elif arg == 1: + screen.erase_up() + elif arg == 2: + screen.erase_screen() + +def DoEraseEndOfLine (fsm): + + screen = fsm.memory[0] + screen.erase_end_of_line() + +def DoEraseLine (fsm): + + arg = int(fsm.memory.pop()) + screen = fsm.memory[0] + if arg == 0: + screen.erase_end_of_line() + elif arg == 1: + screen.erase_start_of_line() + elif arg == 2: + screen.erase_line() + +def DoEnableScroll (fsm): + + screen = fsm.memory[0] + screen.scroll_screen() + +def DoCursorSave (fsm): + + screen = fsm.memory[0] + screen.cursor_save_attrs() + +def DoCursorRestore (fsm): + + screen = fsm.memory[0] + screen.cursor_restore_attrs() + +def DoScrollRegion (fsm): + + screen = fsm.memory[0] + r2 = int(fsm.memory.pop()) + r1 = int(fsm.memory.pop()) + screen.scroll_screen_rows (r1,r2) + +def DoMode (fsm): + + screen = fsm.memory[0] + mode = fsm.memory.pop() # Should be 4 + # screen.setReplaceMode () + +def DoLog (fsm): + + screen = fsm.memory[0] + fsm.memory = [screen] + fout = open ('log', 'a') + fout.write (fsm.input_symbol + ',' + fsm.current_state + '\n') + fout.close() + +class term (screen.screen): + + '''This class is an abstract, generic terminal. + This does nothing. This is a placeholder that + provides a common base class for other terminals + such as an ANSI terminal. ''' + + def __init__ (self, r=24, c=80, *args, **kwargs): + + screen.screen.__init__(self, r,c,*args,**kwargs) + +class ANSI (term): + '''This class implements an ANSI (VT100) terminal. + It is a stream filter that recognizes ANSI terminal + escape sequences and maintains the state of a screen object. ''' + + def __init__ (self, r=24,c=80,*args,**kwargs): + + term.__init__(self,r,c,*args,**kwargs) + + #self.screen = screen (24,80) + self.state = FSM.FSM ('INIT',[self]) + self.state.set_default_transition (DoLog, 'INIT') + self.state.add_transition_any ('INIT', DoEmit, 'INIT') + self.state.add_transition ('\x1b', 'INIT', None, 'ESC') + self.state.add_transition_any ('ESC', DoLog, 'INIT') + self.state.add_transition ('(', 'ESC', None, 'G0SCS') + self.state.add_transition (')', 'ESC', None, 'G1SCS') + self.state.add_transition_list ('AB012', 'G0SCS', None, 'INIT') + self.state.add_transition_list ('AB012', 'G1SCS', None, 'INIT') + self.state.add_transition ('7', 'ESC', DoCursorSave, 'INIT') + self.state.add_transition ('8', 'ESC', DoCursorRestore, 'INIT') + self.state.add_transition ('M', 'ESC', DoUpReverse, 'INIT') + self.state.add_transition ('>', 'ESC', DoUpReverse, 'INIT') + self.state.add_transition ('<', 'ESC', DoUpReverse, 'INIT') + self.state.add_transition ('=', 'ESC', None, 'INIT') # Selects application keypad. + self.state.add_transition ('#', 'ESC', None, 'GRAPHICS_POUND') + self.state.add_transition_any ('GRAPHICS_POUND', None, 'INIT') + self.state.add_transition ('[', 'ESC', None, 'ELB') + # ELB means Escape Left Bracket. That is ^[[ + self.state.add_transition ('H', 'ELB', DoHomeOrigin, 'INIT') + self.state.add_transition ('D', 'ELB', DoBackOne, 'INIT') + self.state.add_transition ('B', 'ELB', DoDownOne, 'INIT') + self.state.add_transition ('C', 'ELB', DoForwardOne, 'INIT') + self.state.add_transition ('A', 'ELB', DoUpOne, 'INIT') + self.state.add_transition ('J', 'ELB', DoEraseDown, 'INIT') + self.state.add_transition ('K', 'ELB', DoEraseEndOfLine, 'INIT') + self.state.add_transition ('r', 'ELB', DoEnableScroll, 'INIT') + self.state.add_transition ('m', 'ELB', self.do_sgr, 'INIT') + self.state.add_transition ('?', 'ELB', None, 'MODECRAP') + self.state.add_transition_list (string.digits, 'ELB', DoStartNumber, 'NUMBER_1') + self.state.add_transition_list (string.digits, 'NUMBER_1', DoBuildNumber, 'NUMBER_1') + self.state.add_transition ('D', 'NUMBER_1', DoBack, 'INIT') + self.state.add_transition ('B', 'NUMBER_1', DoDown, 'INIT') + self.state.add_transition ('C', 'NUMBER_1', DoForward, 'INIT') + self.state.add_transition ('A', 'NUMBER_1', DoUp, 'INIT') + self.state.add_transition ('J', 'NUMBER_1', DoErase, 'INIT') + self.state.add_transition ('K', 'NUMBER_1', DoEraseLine, 'INIT') + self.state.add_transition ('l', 'NUMBER_1', DoMode, 'INIT') + ### It gets worse... the 'm' code can have infinite number of + ### number;number;number before it. I've never seen more than two, + ### but the specs say it's allowed. crap! + self.state.add_transition ('m', 'NUMBER_1', self.do_sgr, 'INIT') + ### LED control. Same implementation problem as 'm' code. + self.state.add_transition ('q', 'NUMBER_1', self.do_decsca, 'INIT') + + # \E[?47h switch to alternate screen + # \E[?47l restores to normal screen from alternate screen. + self.state.add_transition_list (string.digits, 'MODECRAP', DoStartNumber, 'MODECRAP_NUM') + self.state.add_transition_list (string.digits, 'MODECRAP_NUM', DoBuildNumber, 'MODECRAP_NUM') + self.state.add_transition ('l', 'MODECRAP_NUM', self.do_modecrap, 'INIT') + self.state.add_transition ('h', 'MODECRAP_NUM', self.do_modecrap, 'INIT') + +#RM Reset Mode Esc [ Ps l none + self.state.add_transition (';', 'NUMBER_1', None, 'SEMICOLON') + self.state.add_transition_any ('SEMICOLON', DoLog, 'INIT') + self.state.add_transition_list (string.digits, 'SEMICOLON', DoStartNumber, 'NUMBER_2') + self.state.add_transition_list (string.digits, 'NUMBER_2', DoBuildNumber, 'NUMBER_2') + self.state.add_transition_any ('NUMBER_2', DoLog, 'INIT') + self.state.add_transition ('H', 'NUMBER_2', DoHome, 'INIT') + self.state.add_transition ('f', 'NUMBER_2', DoHome, 'INIT') + self.state.add_transition ('r', 'NUMBER_2', DoScrollRegion, 'INIT') + ### It gets worse... the 'm' code can have infinite number of + ### number;number;number before it. I've never seen more than two, + ### but the specs say it's allowed. crap! + self.state.add_transition ('m', 'NUMBER_2', self.do_sgr, 'INIT') + ### LED control. Same problem as 'm' code. + self.state.add_transition ('q', 'NUMBER_2', self.do_decsca, 'INIT') + self.state.add_transition (';', 'NUMBER_2', None, 'SEMICOLON_X') + + # Create a state for 'q' and 'm' which allows an infinite number of ignored numbers + self.state.add_transition_any ('SEMICOLON_X', DoLog, 'INIT') + self.state.add_transition_list (string.digits, 'SEMICOLON_X', DoStartNumber, 'NUMBER_X') + self.state.add_transition_list (string.digits, 'NUMBER_X', DoBuildNumber, 'NUMBER_X') + self.state.add_transition_any ('NUMBER_X', DoLog, 'INIT') + self.state.add_transition ('m', 'NUMBER_X', self.do_sgr, 'INIT') + self.state.add_transition ('q', 'NUMBER_X', self.do_decsca, 'INIT') + self.state.add_transition (';', 'NUMBER_X', None, 'SEMICOLON_X') + + def process (self, c): + """Process a single character. Called by :meth:`write`.""" + if isinstance(c, bytes): + c = self._decode(c) + self.state.process(c) + + def process_list (self, l): + + self.write(l) + + def write (self, s): + """Process text, writing it to the virtual screen while handling + ANSI escape codes. + """ + if isinstance(s, bytes): + s = self._decode(s) + for c in s: + self.process(c) + + def flush (self): + pass + + def write_ch (self, ch): + '''This puts a character at the current cursor position. The cursor + position is moved forward with wrap-around, but no scrolling is done if + the cursor hits the lower-right corner of the screen. ''' + + if isinstance(ch, bytes): + ch = self._decode(ch) + + #\r and \n both produce a call to cr() and lf(), respectively. + ch = ch[0] + + if ch == u'\r': + self.cr() + return + if ch == u'\n': + self.crlf() + return + if ch == chr(screen.BS): + self.cursor_back() + return + self.put_abs(self.cur_r, self.cur_c, ch) + old_r = self.cur_r + old_c = self.cur_c + self.cursor_forward() + if old_c == self.cur_c: + self.cursor_down() + if old_r != self.cur_r: + self.cursor_home (self.cur_r, 1) + else: + self.scroll_up () + self.cursor_home (self.cur_r, 1) + self.erase_line() + + def do_sgr (self, fsm): + '''Select Graphic Rendition, e.g. color. ''' + screen = fsm.memory[0] + fsm.memory = [screen] + + def do_decsca (self, fsm): + '''Select character protection attribute. ''' + screen = fsm.memory[0] + fsm.memory = [screen] + + def do_modecrap (self, fsm): + '''Handler for \x1b[?h and \x1b[?l. If anyone + wanted to actually use these, they'd need to add more states to the + FSM rather than just improve or override this method. ''' + screen = fsm.memory[0] + fsm.memory = [screen] diff --git a/lib/python3.12/site-packages/pexpect/FSM.py b/lib/python3.12/site-packages/pexpect/FSM.py new file mode 100644 index 0000000000000000000000000000000000000000..46b392ea08aaf53577b27c0552776ecc86d72510 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/FSM.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python + +'''This module implements a Finite State Machine (FSM). In addition to state +this FSM also maintains a user defined "memory". So this FSM can be used as a +Push-down Automata (PDA) since a PDA is a FSM + memory. + +The following describes how the FSM works, but you will probably also need to +see the example function to understand how the FSM is used in practice. + +You define an FSM by building tables of transitions. For a given input symbol +the process() method uses these tables to decide what action to call and what +the next state will be. The FSM has a table of transitions that associate: + + (input_symbol, current_state) --> (action, next_state) + +Where "action" is a function you define. The symbols and states can be any +objects. You use the add_transition() and add_transition_list() methods to add +to the transition table. The FSM also has a table of transitions that +associate: + + (current_state) --> (action, next_state) + +You use the add_transition_any() method to add to this transition table. The +FSM also has one default transition that is not associated with any specific +input_symbol or state. You use the set_default_transition() method to set the +default transition. + +When an action function is called it is passed a reference to the FSM. The +action function may then access attributes of the FSM such as input_symbol, +current_state, or "memory". The "memory" attribute can be any object that you +want to pass along to the action functions. It is not used by the FSM itself. +For parsing you would typically pass a list to be used as a stack. + +The processing sequence is as follows. The process() method is given an +input_symbol to process. The FSM will search the table of transitions that +associate: + + (input_symbol, current_state) --> (action, next_state) + +If the pair (input_symbol, current_state) is found then process() will call the +associated action function and then set the current state to the next_state. + +If the FSM cannot find a match for (input_symbol, current_state) it will then +search the table of transitions that associate: + + (current_state) --> (action, next_state) + +If the current_state is found then the process() method will call the +associated action function and then set the current state to the next_state. +Notice that this table lacks an input_symbol. It lets you define transitions +for a current_state and ANY input_symbol. Hence, it is called the "any" table. +Remember, it is always checked after first searching the table for a specific +(input_symbol, current_state). + +For the case where the FSM did not match either of the previous two cases the +FSM will try to use the default transition. If the default transition is +defined then the process() method will call the associated action function and +then set the current state to the next_state. This lets you define a default +transition as a catch-all case. You can think of it as an exception handler. +There can be only one default transition. + +Finally, if none of the previous cases are defined for an input_symbol and +current_state then the FSM will raise an exception. This may be desirable, but +you can always prevent this just by defining a default transition. + +Noah Spurrier 20020822 + +PEXPECT LICENSE + + This license is approved by the OSI and FSF as GPL-compatible. + http://opensource.org/licenses/isc-license.txt + + Copyright (c) 2012, Noah Spurrier + PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY + PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE + COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +''' + +class ExceptionFSM(Exception): + + '''This is the FSM Exception class.''' + + def __init__(self, value): + self.value = value + + def __str__(self): + return 'ExceptionFSM: ' + str(self.value) + +class FSM: + + '''This is a Finite State Machine (FSM). + ''' + + def __init__(self, initial_state, memory=None): + + '''This creates the FSM. You set the initial state here. The "memory" + attribute is any object that you want to pass along to the action + functions. It is not used by the FSM. For parsing you would typically + pass a list to be used as a stack. ''' + + # Map (input_symbol, current_state) --> (action, next_state). + self.state_transitions = {} + # Map (current_state) --> (action, next_state). + self.state_transitions_any = {} + self.default_transition = None + + self.input_symbol = None + self.initial_state = initial_state + self.current_state = self.initial_state + self.next_state = None + self.action = None + self.memory = memory + + def reset (self): + + '''This sets the current_state to the initial_state and sets + input_symbol to None. The initial state was set by the constructor + __init__(). ''' + + self.current_state = self.initial_state + self.input_symbol = None + + def add_transition (self, input_symbol, state, action=None, next_state=None): + + '''This adds a transition that associates: + + (input_symbol, current_state) --> (action, next_state) + + The action may be set to None in which case the process() method will + ignore the action and only set the next_state. The next_state may be + set to None in which case the current state will be unchanged. + + You can also set transitions for a list of symbols by using + add_transition_list(). ''' + + if next_state is None: + next_state = state + self.state_transitions[(input_symbol, state)] = (action, next_state) + + def add_transition_list (self, list_input_symbols, state, action=None, next_state=None): + + '''This adds the same transition for a list of input symbols. + You can pass a list or a string. Note that it is handy to use + string.digits, string.whitespace, string.letters, etc. to add + transitions that match character classes. + + The action may be set to None in which case the process() method will + ignore the action and only set the next_state. The next_state may be + set to None in which case the current state will be unchanged. ''' + + if next_state is None: + next_state = state + for input_symbol in list_input_symbols: + self.add_transition (input_symbol, state, action, next_state) + + def add_transition_any (self, state, action=None, next_state=None): + + '''This adds a transition that associates: + + (current_state) --> (action, next_state) + + That is, any input symbol will match the current state. + The process() method checks the "any" state associations after it first + checks for an exact match of (input_symbol, current_state). + + The action may be set to None in which case the process() method will + ignore the action and only set the next_state. The next_state may be + set to None in which case the current state will be unchanged. ''' + + if next_state is None: + next_state = state + self.state_transitions_any [state] = (action, next_state) + + def set_default_transition (self, action, next_state): + + '''This sets the default transition. This defines an action and + next_state if the FSM cannot find the input symbol and the current + state in the transition list and if the FSM cannot find the + current_state in the transition_any list. This is useful as a final + fall-through state for catching errors and undefined states. + + The default transition can be removed by setting the attribute + default_transition to None. ''' + + self.default_transition = (action, next_state) + + def get_transition (self, input_symbol, state): + + '''This returns (action, next state) given an input_symbol and state. + This does not modify the FSM state, so calling this method has no side + effects. Normally you do not call this method directly. It is called by + process(). + + The sequence of steps to check for a defined transition goes from the + most specific to the least specific. + + 1. Check state_transitions[] that match exactly the tuple, + (input_symbol, state) + + 2. Check state_transitions_any[] that match (state) + In other words, match a specific state and ANY input_symbol. + + 3. Check if the default_transition is defined. + This catches any input_symbol and any state. + This is a handler for errors, undefined states, or defaults. + + 4. No transition was defined. If we get here then raise an exception. + ''' + + if (input_symbol, state) in self.state_transitions: + return self.state_transitions[(input_symbol, state)] + elif state in self.state_transitions_any: + return self.state_transitions_any[state] + elif self.default_transition is not None: + return self.default_transition + else: + raise ExceptionFSM ('Transition is undefined: (%s, %s).' % + (str(input_symbol), str(state)) ) + + def process (self, input_symbol): + + '''This is the main method that you call to process input. This may + cause the FSM to change state and call an action. This method calls + get_transition() to find the action and next_state associated with the + input_symbol and current_state. If the action is None then the action + is not called and only the current state is changed. This method + processes one complete input symbol. You can process a list of symbols + (or a string) by calling process_list(). ''' + + self.input_symbol = input_symbol + (self.action, self.next_state) = self.get_transition (self.input_symbol, self.current_state) + if self.action is not None: + self.action (self) + self.current_state = self.next_state + self.next_state = None + + def process_list (self, input_symbols): + + '''This takes a list and sends each element to process(). The list may + be a string or any iterable object. ''' + + for s in input_symbols: + self.process (s) + +############################################################################## +# The following is an example that demonstrates the use of the FSM class to +# process an RPN expression. Run this module from the command line. You will +# get a prompt > for input. Enter an RPN Expression. Numbers may be integers. +# Operators are * / + - Use the = sign to evaluate and print the expression. +# For example: +# +# 167 3 2 2 * * * 1 - = +# +# will print: +# +# 2003 +############################################################################## + +import sys +import string + +PY3 = (sys.version_info[0] >= 3) + +# +# These define the actions. +# Note that "memory" is a list being used as a stack. +# + +def BeginBuildNumber (fsm): + fsm.memory.append (fsm.input_symbol) + +def BuildNumber (fsm): + s = fsm.memory.pop () + s = s + fsm.input_symbol + fsm.memory.append (s) + +def EndBuildNumber (fsm): + s = fsm.memory.pop () + fsm.memory.append (int(s)) + +def DoOperator (fsm): + ar = fsm.memory.pop() + al = fsm.memory.pop() + if fsm.input_symbol == '+': + fsm.memory.append (al + ar) + elif fsm.input_symbol == '-': + fsm.memory.append (al - ar) + elif fsm.input_symbol == '*': + fsm.memory.append (al * ar) + elif fsm.input_symbol == '/': + fsm.memory.append (al / ar) + +def DoEqual (fsm): + print(str(fsm.memory.pop())) + +def Error (fsm): + print('That does not compute.') + print(str(fsm.input_symbol)) + +def main(): + + '''This is where the example starts and the FSM state transitions are + defined. Note that states are strings (such as 'INIT'). This is not + necessary, but it makes the example easier to read. ''' + + f = FSM ('INIT', []) + f.set_default_transition (Error, 'INIT') + f.add_transition_any ('INIT', None, 'INIT') + f.add_transition ('=', 'INIT', DoEqual, 'INIT') + f.add_transition_list (string.digits, 'INIT', BeginBuildNumber, 'BUILDING_NUMBER') + f.add_transition_list (string.digits, 'BUILDING_NUMBER', BuildNumber, 'BUILDING_NUMBER') + f.add_transition_list (string.whitespace, 'BUILDING_NUMBER', EndBuildNumber, 'INIT') + f.add_transition_list ('+-*/', 'INIT', DoOperator, 'INIT') + + print() + print('Enter an RPN Expression.') + print('Numbers may be integers. Operators are * / + -') + print('Use the = sign to evaluate and print the expression.') + print('For example: ') + print(' 167 3 2 2 * * * 1 - =') + inputstr = (input if PY3 else raw_input)('> ') # analysis:ignore + f.process_list(inputstr) + + +if __name__ == '__main__': + main() diff --git a/lib/python3.12/site-packages/pexpect/__init__.py b/lib/python3.12/site-packages/pexpect/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..86254ee720ecda4e15e7656dc0c0e0febd61d291 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/__init__.py @@ -0,0 +1,91 @@ +'''Pexpect is a Python module for spawning child applications and controlling +them automatically. Pexpect can be used for automating interactive applications +such as ssh, ftp, passwd, telnet, etc. It can be used to automate setup +scripts for duplicating software package installations on different servers. It +can be used for automated software testing. Pexpect is in the spirit of Don +Libes' Expect, but Pexpect is pure Python. Other Expect-like modules for Python +require TCL and Expect or require C extensions to be compiled. Pexpect does not +use C, Expect, or TCL extensions. It should work on any platform that supports +the standard Python pty module. The Pexpect interface focuses on ease of use so +that simple tasks are easy. + +There are two main interfaces to the Pexpect system; these are the function, +run() and the class, spawn. The spawn class is more powerful. The run() +function is simpler than spawn, and is good for quickly calling program. When +you call the run() function it executes a given program and then returns the +output. This is a handy replacement for os.system(). + +For example:: + + pexpect.run('ls -la') + +The spawn class is the more powerful interface to the Pexpect system. You can +use this to spawn a child program then interact with it by sending input and +expecting responses (waiting for patterns in the child's output). + +For example:: + + child = pexpect.spawn('scp foo user@example.com:.') + child.expect('Password:') + child.sendline(mypassword) + +Context manager can be used for the spawn() function:: + + with pexpect.spawn('scp foo user@example.com:.') as child: + child.expect('Password:') + child.sendline(mypassword) + +This works even for commands that ask for passwords or other input outside of +the normal stdio streams. For example, ssh reads input directly from the TTY +device which bypasses stdin. + +Credits: Noah Spurrier, Richard Holden, Marco Molteni, Kimberley Burchett, +Robert Stone, Hartmut Goebel, Chad Schroeder, Erick Tryzelaar, Dave Kirby, Ids +vander Molen, George Todd, Noel Taylor, Nicolas D. Cesar, Alexander Gattin, +Jacques-Etienne Baudoux, Geoffrey Marshall, Francisco Lourenco, Glen Mabey, +Karthik Gurusamy, Fernando Perez, Corey Minyard, Jon Cohen, Guillaume +Chazarain, Andrew Ryan, Nick Craig-Wood, Andrew Stone, Jorgen Grahn, John +Spiegel, Jan Grant, and Shane Kerr. Let me know if I forgot anyone. + +Pexpect is free, open source, and all that good stuff. +http://pexpect.sourceforge.net/ + +PEXPECT LICENSE + + This license is approved by the OSI and FSF as GPL-compatible. + http://opensource.org/licenses/isc-license.txt + + Copyright (c) 2012, Noah Spurrier + PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY + PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE + COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +''' + +import sys +PY3 = (sys.version_info[0] >= 3) + +from .exceptions import ExceptionPexpect, EOF, TIMEOUT +from .utils import split_command_line, which, is_executable_file +from .expect import Expecter, searcher_re, searcher_string + +if sys.platform != 'win32': + # On Unix, these are available at the top level for backwards compatibility + from .pty_spawn import spawn, spawnu + from .run import run, runu + +__version__ = '4.9.0' +__revision__ = '' +__all__ = ['ExceptionPexpect', 'EOF', 'TIMEOUT', 'spawn', 'spawnu', 'run', 'runu', + 'which', 'split_command_line', '__version__', '__revision__'] + + + +# vim: set shiftround expandtab tabstop=4 shiftwidth=4 ft=python autoindent : diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/ANSI.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/ANSI.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ecaf542853dfd6a8c1d1c267b01bb3c3526fc6cd Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/ANSI.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/FSM.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/FSM.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..51da88909bd81024be6f46a0a339ca7972938e3b Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/FSM.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6ce02a9f823e34aa5ced0d54746463f66d9f65d2 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/_async.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/_async.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..87b41ccfa4d31e6cbe7749617a949780e529aeef Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/_async.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/_async_pre_await.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/_async_pre_await.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b22b4b8bebab3aeef74a41116818f95e93bd300 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/_async_pre_await.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/_async_w_await.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/_async_w_await.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8952521327148b5d032fc2bfc33180b6e55b41de Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/_async_w_await.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/exceptions.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/exceptions.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..695ebe577ab8297653aa6312bcce0f61c18fc43e Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/exceptions.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/expect.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/expect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..804299156ad4eb1b311e8a94c1fc88a1c1ae36a8 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/expect.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/fdpexpect.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/fdpexpect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..40e80fa43b72232c4e19d948d5331d073f0e2652 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/fdpexpect.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/popen_spawn.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/popen_spawn.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..392f1bdf6a6dc593b078a5fc94d1e6190259a3e6 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/popen_spawn.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/pty_spawn.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/pty_spawn.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fa9db424b09884959cb08ccfb2daa56f127bdfe8 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/pty_spawn.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/pxssh.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/pxssh.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..545cf0b544340d54e90d522e5c2f4e5af7c5d057 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/pxssh.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/replwrap.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/replwrap.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a93df92f2f595438e1b43a52faee5bc692201d93 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/replwrap.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/run.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/run.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ca9a176329942acc5561b9b1a98d83494cfc757f Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/run.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/screen.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/screen.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..77aac32f5879b6c8d47474874a358d23aaeccebc Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/screen.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/socket_pexpect.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/socket_pexpect.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5f2d031a4964966111b4aede95a7138827489707 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/socket_pexpect.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/spawnbase.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/spawnbase.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf5bf186dbaecc8107e326c3f3f12913865e7340 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/spawnbase.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/__pycache__/utils.cpython-312.pyc b/lib/python3.12/site-packages/pexpect/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..140f96a11fbb3db211332092621183218c207353 Binary files /dev/null and b/lib/python3.12/site-packages/pexpect/__pycache__/utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/pexpect/_async.py b/lib/python3.12/site-packages/pexpect/_async.py new file mode 100644 index 0000000000000000000000000000000000000000..261720c16069e89411b1511eeba53f1068c3c913 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/_async.py @@ -0,0 +1,28 @@ +"""Facade that provides coroutines implementation pertinent to running Py version. + +Python 3.5 introduced the async def/await syntax keyword. +With later versions coroutines and methods to get the running asyncio loop are +being deprecated, not supported anymore. + +For Python versions later than 3.6, coroutines and objects that are defined via +``async def``/``await`` keywords are imported. + +Here the code is just imported, to provide the same interface to older code. +""" +# pylint: disable=unused-import +# flake8: noqa: F401 +from sys import version_info as py_version_info + +# this assumes async def/await are more stable +if py_version_info >= (3, 6): + from pexpect._async_w_await import ( + PatternWaiter, + expect_async, + repl_run_command_async, + ) +else: + from pexpect._async_pre_await import ( + PatternWaiter, + expect_async, + repl_run_command_async, + ) diff --git a/lib/python3.12/site-packages/pexpect/_async_pre_await.py b/lib/python3.12/site-packages/pexpect/_async_pre_await.py new file mode 100644 index 0000000000000000000000000000000000000000..81ece1b6da1c9effe768717844ebc791ce7b86a5 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/_async_pre_await.py @@ -0,0 +1,111 @@ +"""Implementation of coroutines without using ``async def``/``await`` keywords. + +``@asyncio.coroutine`` and ``yield from`` are used here instead. +""" +import asyncio +import errno +import signal + +from pexpect import EOF + + +@asyncio.coroutine +def expect_async(expecter, timeout=None): + # First process data that was previously read - if it maches, we don't need + # async stuff. + idx = expecter.existing_data() + if idx is not None: + return idx + if not expecter.spawn.async_pw_transport: + pw = PatternWaiter() + pw.set_expecter(expecter) + transport, pw = yield from asyncio.get_event_loop().connect_read_pipe( + lambda: pw, expecter.spawn + ) + expecter.spawn.async_pw_transport = pw, transport + else: + pw, transport = expecter.spawn.async_pw_transport + pw.set_expecter(expecter) + transport.resume_reading() + try: + return (yield from asyncio.wait_for(pw.fut, timeout)) + except asyncio.TimeoutError as e: + transport.pause_reading() + return expecter.timeout(e) + + +@asyncio.coroutine +def repl_run_command_async(repl, cmdlines, timeout=-1): + res = [] + repl.child.sendline(cmdlines[0]) + for line in cmdlines[1:]: + yield from repl._expect_prompt(timeout=timeout, async_=True) + res.append(repl.child.before) + repl.child.sendline(line) + + # Command was fully submitted, now wait for the next prompt + prompt_idx = yield from repl._expect_prompt(timeout=timeout, async_=True) + if prompt_idx == 1: + # We got the continuation prompt - command was incomplete + repl.child.kill(signal.SIGINT) + yield from repl._expect_prompt(timeout=1, async_=True) + raise ValueError("Continuation prompt found - input was incomplete:") + return "".join(res + [repl.child.before]) + + +class PatternWaiter(asyncio.Protocol): + transport = None + + def set_expecter(self, expecter): + self.expecter = expecter + self.fut = asyncio.Future() + + def found(self, result): + if not self.fut.done(): + self.fut.set_result(result) + self.transport.pause_reading() + + def error(self, exc): + if not self.fut.done(): + self.fut.set_exception(exc) + self.transport.pause_reading() + + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + spawn = self.expecter.spawn + s = spawn._decoder.decode(data) + spawn._log(s, "read") + + if self.fut.done(): + spawn._before.write(s) + spawn._buffer.write(s) + return + + try: + index = self.expecter.new_data(s) + if index is not None: + # Found a match + self.found(index) + except Exception as e: + self.expecter.errored() + self.error(e) + + def eof_received(self): + # N.B. If this gets called, async will close the pipe (the spawn object) + # for us + try: + self.expecter.spawn.flag_eof = True + index = self.expecter.eof() + except EOF as e: + self.error(e) + else: + self.found(index) + + def connection_lost(self, exc): + if isinstance(exc, OSError) and exc.errno == errno.EIO: + # We may get here without eof_received being called, e.g on Linux + self.eof_received() + elif exc is not None: + self.error(exc) diff --git a/lib/python3.12/site-packages/pexpect/_async_w_await.py b/lib/python3.12/site-packages/pexpect/_async_w_await.py new file mode 100644 index 0000000000000000000000000000000000000000..59cb1ef13d5041ddf24398891db9fa41c2ad63d2 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/_async_w_await.py @@ -0,0 +1,118 @@ +"""Implementation of coroutines using ``async def``/``await`` keywords. + +These keywords replaced ``@asyncio.coroutine`` and ``yield from`` from +Python 3.5 onwards. +""" +import asyncio +import errno +import signal +from sys import version_info as py_version_info + +from pexpect import EOF + +if py_version_info >= (3, 7): + # get_running_loop, new in 3.7, is preferred to get_event_loop + _loop_getter = asyncio.get_running_loop +else: + # Deprecation warning since 3.10 + _loop_getter = asyncio.get_event_loop + + +async def expect_async(expecter, timeout=None): + # First process data that was previously read - if it maches, we don't need + # async stuff. + idx = expecter.existing_data() + if idx is not None: + return idx + if not expecter.spawn.async_pw_transport: + pattern_waiter = PatternWaiter() + pattern_waiter.set_expecter(expecter) + transport, pattern_waiter = await _loop_getter().connect_read_pipe( + lambda: pattern_waiter, expecter.spawn + ) + expecter.spawn.async_pw_transport = pattern_waiter, transport + else: + pattern_waiter, transport = expecter.spawn.async_pw_transport + pattern_waiter.set_expecter(expecter) + transport.resume_reading() + try: + return await asyncio.wait_for(pattern_waiter.fut, timeout) + except asyncio.TimeoutError as exc: + transport.pause_reading() + return expecter.timeout(exc) + + +async def repl_run_command_async(repl, cmdlines, timeout=-1): + res = [] + repl.child.sendline(cmdlines[0]) + for line in cmdlines[1:]: + await repl._expect_prompt(timeout=timeout, async_=True) + res.append(repl.child.before) + repl.child.sendline(line) + + # Command was fully submitted, now wait for the next prompt + prompt_idx = await repl._expect_prompt(timeout=timeout, async_=True) + if prompt_idx == 1: + # We got the continuation prompt - command was incomplete + repl.child.kill(signal.SIGINT) + await repl._expect_prompt(timeout=1, async_=True) + raise ValueError("Continuation prompt found - input was incomplete:") + return "".join(res + [repl.child.before]) + + +class PatternWaiter(asyncio.Protocol): + transport = None + + def set_expecter(self, expecter): + self.expecter = expecter + self.fut = asyncio.Future() + + def found(self, result): + if not self.fut.done(): + self.fut.set_result(result) + self.transport.pause_reading() + + def error(self, exc): + if not self.fut.done(): + self.fut.set_exception(exc) + self.transport.pause_reading() + + def connection_made(self, transport): + self.transport = transport + + def data_received(self, data): + spawn = self.expecter.spawn + s = spawn._decoder.decode(data) + spawn._log(s, "read") + + if self.fut.done(): + spawn._before.write(s) + spawn._buffer.write(s) + return + + try: + index = self.expecter.new_data(s) + if index is not None: + # Found a match + self.found(index) + except Exception as exc: + self.expecter.errored() + self.error(exc) + + def eof_received(self): + # N.B. If this gets called, async will close the pipe (the spawn object) + # for us + try: + self.expecter.spawn.flag_eof = True + index = self.expecter.eof() + except EOF as exc: + self.error(exc) + else: + self.found(index) + + def connection_lost(self, exc): + if isinstance(exc, OSError) and exc.errno == errno.EIO: + # We may get here without eof_received being called, e.g on Linux + self.eof_received() + elif exc is not None: + self.error(exc) diff --git a/lib/python3.12/site-packages/pexpect/bashrc.sh b/lib/python3.12/site-packages/pexpect/bashrc.sh new file mode 100644 index 0000000000000000000000000000000000000000..d75d1a5b626e4c6c12b17655e089a1ecc72fc70e --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/bashrc.sh @@ -0,0 +1,18 @@ +# Different platforms have different names for the systemwide bashrc +if [[ -f /etc/bashrc ]]; then + source /etc/bashrc +fi +if [[ -f /etc/bash.bashrc ]]; then + source /etc/bash.bashrc +fi +if [[ -f ~/.bashrc ]]; then + source ~/.bashrc +fi + +# Reset PS1 so pexpect can find it +PS1="$" + +# Unset PROMPT_COMMAND, so that it can't change PS1 to something unexpected. +unset PROMPT_COMMAND + +bind 'set enable-bracketed-paste off' diff --git a/lib/python3.12/site-packages/pexpect/exceptions.py b/lib/python3.12/site-packages/pexpect/exceptions.py new file mode 100644 index 0000000000000000000000000000000000000000..cb360f02614304b306714b309cbc9b0a0f2ff770 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/exceptions.py @@ -0,0 +1,35 @@ +"""Exception classes used by Pexpect""" + +import traceback +import sys + +class ExceptionPexpect(Exception): + '''Base class for all exceptions raised by this module. + ''' + + def __init__(self, value): + super(ExceptionPexpect, self).__init__(value) + self.value = value + + def __str__(self): + return str(self.value) + + def get_trace(self): + '''This returns an abbreviated stack trace with lines that only concern + the caller. In other words, the stack trace inside the Pexpect module + is not included. ''' + + tblist = traceback.extract_tb(sys.exc_info()[2]) + tblist = [item for item in tblist if ('pexpect/__init__' not in item[0]) + and ('pexpect/expect' not in item[0])] + tblist = traceback.format_list(tblist) + return ''.join(tblist) + + +class EOF(ExceptionPexpect): + '''Raised when EOF is read from a child. + This usually means the child has exited.''' + + +class TIMEOUT(ExceptionPexpect): + '''Raised when a read time exceeds the timeout. ''' diff --git a/lib/python3.12/site-packages/pexpect/expect.py b/lib/python3.12/site-packages/pexpect/expect.py new file mode 100644 index 0000000000000000000000000000000000000000..d3409db9d73d55d5b59fa15816be1021af1439bc --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/expect.py @@ -0,0 +1,371 @@ +import time + +from .exceptions import EOF, TIMEOUT + +class Expecter(object): + def __init__(self, spawn, searcher, searchwindowsize=-1): + self.spawn = spawn + self.searcher = searcher + # A value of -1 means to use the figure from spawn, which should + # be None or a positive number. + if searchwindowsize == -1: + searchwindowsize = spawn.searchwindowsize + self.searchwindowsize = searchwindowsize + self.lookback = None + if hasattr(searcher, 'longest_string'): + self.lookback = searcher.longest_string + + def do_search(self, window, freshlen): + spawn = self.spawn + searcher = self.searcher + if freshlen > len(window): + freshlen = len(window) + index = searcher.search(window, freshlen, self.searchwindowsize) + if index >= 0: + spawn._buffer = spawn.buffer_type() + spawn._buffer.write(window[searcher.end:]) + spawn.before = spawn._before.getvalue()[ + 0:-(len(window) - searcher.start)] + spawn._before = spawn.buffer_type() + spawn._before.write(window[searcher.end:]) + spawn.after = window[searcher.start:searcher.end] + spawn.match = searcher.match + spawn.match_index = index + # Found a match + return index + elif self.searchwindowsize or self.lookback: + maintain = self.searchwindowsize or self.lookback + if spawn._buffer.tell() > maintain: + spawn._buffer = spawn.buffer_type() + spawn._buffer.write(window[-maintain:]) + + def existing_data(self): + # First call from a new call to expect_loop or expect_async. + # self.searchwindowsize may have changed. + # Treat all data as fresh. + spawn = self.spawn + before_len = spawn._before.tell() + buf_len = spawn._buffer.tell() + freshlen = before_len + if before_len > buf_len: + if not self.searchwindowsize: + spawn._buffer = spawn.buffer_type() + window = spawn._before.getvalue() + spawn._buffer.write(window) + elif buf_len < self.searchwindowsize: + spawn._buffer = spawn.buffer_type() + spawn._before.seek( + max(0, before_len - self.searchwindowsize)) + window = spawn._before.read() + spawn._buffer.write(window) + else: + spawn._buffer.seek(max(0, buf_len - self.searchwindowsize)) + window = spawn._buffer.read() + else: + if self.searchwindowsize: + spawn._buffer.seek(max(0, buf_len - self.searchwindowsize)) + window = spawn._buffer.read() + else: + window = spawn._buffer.getvalue() + return self.do_search(window, freshlen) + + def new_data(self, data): + # A subsequent call, after a call to existing_data. + spawn = self.spawn + freshlen = len(data) + spawn._before.write(data) + if not self.searchwindowsize: + if self.lookback: + # search lookback + new data. + old_len = spawn._buffer.tell() + spawn._buffer.write(data) + spawn._buffer.seek(max(0, old_len - self.lookback)) + window = spawn._buffer.read() + else: + # copy the whole buffer (really slow for large datasets). + spawn._buffer.write(data) + window = spawn.buffer + else: + if len(data) >= self.searchwindowsize or not spawn._buffer.tell(): + window = data[-self.searchwindowsize:] + spawn._buffer = spawn.buffer_type() + spawn._buffer.write(window[-self.searchwindowsize:]) + else: + spawn._buffer.write(data) + new_len = spawn._buffer.tell() + spawn._buffer.seek(max(0, new_len - self.searchwindowsize)) + window = spawn._buffer.read() + return self.do_search(window, freshlen) + + def eof(self, err=None): + spawn = self.spawn + + spawn.before = spawn._before.getvalue() + spawn._buffer = spawn.buffer_type() + spawn._before = spawn.buffer_type() + spawn.after = EOF + index = self.searcher.eof_index + if index >= 0: + spawn.match = EOF + spawn.match_index = index + return index + else: + spawn.match = None + spawn.match_index = None + msg = str(spawn) + msg += '\nsearcher: %s' % self.searcher + if err is not None: + msg = str(err) + '\n' + msg + + exc = EOF(msg) + exc.__cause__ = None # in Python 3.x we can use "raise exc from None" + raise exc + + def timeout(self, err=None): + spawn = self.spawn + + spawn.before = spawn._before.getvalue() + spawn.after = TIMEOUT + index = self.searcher.timeout_index + if index >= 0: + spawn.match = TIMEOUT + spawn.match_index = index + return index + else: + spawn.match = None + spawn.match_index = None + msg = str(spawn) + msg += '\nsearcher: %s' % self.searcher + if err is not None: + msg = str(err) + '\n' + msg + + exc = TIMEOUT(msg) + exc.__cause__ = None # in Python 3.x we can use "raise exc from None" + raise exc + + def errored(self): + spawn = self.spawn + spawn.before = spawn._before.getvalue() + spawn.after = None + spawn.match = None + spawn.match_index = None + + def expect_loop(self, timeout=-1): + """Blocking expect""" + spawn = self.spawn + + if timeout is not None: + end_time = time.time() + timeout + + try: + idx = self.existing_data() + if idx is not None: + return idx + while True: + # No match at this point + if (timeout is not None) and (timeout < 0): + return self.timeout() + # Still have time left, so read more data + incoming = spawn.read_nonblocking(spawn.maxread, timeout) + if self.spawn.delayafterread is not None: + time.sleep(self.spawn.delayafterread) + idx = self.new_data(incoming) + # Keep reading until exception or return. + if idx is not None: + return idx + if timeout is not None: + timeout = end_time - time.time() + except EOF as e: + return self.eof(e) + except TIMEOUT as e: + return self.timeout(e) + except: + self.errored() + raise + + +class searcher_string(object): + '''This is a plain string search helper for the spawn.expect_any() method. + This helper class is for speed. For more powerful regex patterns + see the helper class, searcher_re. + + Attributes: + + eof_index - index of EOF, or -1 + timeout_index - index of TIMEOUT, or -1 + + After a successful match by the search() method the following attributes + are available: + + start - index into the buffer, first byte of match + end - index into the buffer, first byte after match + match - the matching string itself + + ''' + + def __init__(self, strings): + '''This creates an instance of searcher_string. This argument 'strings' + may be a list; a sequence of strings; or the EOF or TIMEOUT types. ''' + + self.eof_index = -1 + self.timeout_index = -1 + self._strings = [] + self.longest_string = 0 + for n, s in enumerate(strings): + if s is EOF: + self.eof_index = n + continue + if s is TIMEOUT: + self.timeout_index = n + continue + self._strings.append((n, s)) + if len(s) > self.longest_string: + self.longest_string = len(s) + + def __str__(self): + '''This returns a human-readable string that represents the state of + the object.''' + + ss = [(ns[0], ' %d: %r' % ns) for ns in self._strings] + ss.append((-1, 'searcher_string:')) + if self.eof_index >= 0: + ss.append((self.eof_index, ' %d: EOF' % self.eof_index)) + if self.timeout_index >= 0: + ss.append((self.timeout_index, + ' %d: TIMEOUT' % self.timeout_index)) + ss.sort() + ss = list(zip(*ss))[1] + return '\n'.join(ss) + + def search(self, buffer, freshlen, searchwindowsize=None): + '''This searches 'buffer' for the first occurrence of one of the search + strings. 'freshlen' must indicate the number of bytes at the end of + 'buffer' which have not been searched before. It helps to avoid + searching the same, possibly big, buffer over and over again. + + See class spawn for the 'searchwindowsize' argument. + + If there is a match this returns the index of that string, and sets + 'start', 'end' and 'match'. Otherwise, this returns -1. ''' + + first_match = None + + # 'freshlen' helps a lot here. Further optimizations could + # possibly include: + # + # using something like the Boyer-Moore Fast String Searching + # Algorithm; pre-compiling the search through a list of + # strings into something that can scan the input once to + # search for all N strings; realize that if we search for + # ['bar', 'baz'] and the input is '...foo' we need not bother + # rescanning until we've read three more bytes. + # + # Sadly, I don't know enough about this interesting topic. /grahn + + for index, s in self._strings: + if searchwindowsize is None: + # the match, if any, can only be in the fresh data, + # or at the very end of the old data + offset = -(freshlen + len(s)) + else: + # better obey searchwindowsize + offset = -searchwindowsize + n = buffer.find(s, offset) + if n >= 0 and (first_match is None or n < first_match): + first_match = n + best_index, best_match = index, s + if first_match is None: + return -1 + self.match = best_match + self.start = first_match + self.end = self.start + len(self.match) + return best_index + + +class searcher_re(object): + '''This is regular expression string search helper for the + spawn.expect_any() method. This helper class is for powerful + pattern matching. For speed, see the helper class, searcher_string. + + Attributes: + + eof_index - index of EOF, or -1 + timeout_index - index of TIMEOUT, or -1 + + After a successful match by the search() method the following attributes + are available: + + start - index into the buffer, first byte of match + end - index into the buffer, first byte after match + match - the re.match object returned by a successful re.search + + ''' + + def __init__(self, patterns): + '''This creates an instance that searches for 'patterns' Where + 'patterns' may be a list or other sequence of compiled regular + expressions, or the EOF or TIMEOUT types.''' + + self.eof_index = -1 + self.timeout_index = -1 + self._searches = [] + for n, s in enumerate(patterns): + if s is EOF: + self.eof_index = n + continue + if s is TIMEOUT: + self.timeout_index = n + continue + self._searches.append((n, s)) + + def __str__(self): + '''This returns a human-readable string that represents the state of + the object.''' + + #ss = [(n, ' %d: re.compile("%s")' % + # (n, repr(s.pattern))) for n, s in self._searches] + ss = list() + for n, s in self._searches: + ss.append((n, ' %d: re.compile(%r)' % (n, s.pattern))) + ss.append((-1, 'searcher_re:')) + if self.eof_index >= 0: + ss.append((self.eof_index, ' %d: EOF' % self.eof_index)) + if self.timeout_index >= 0: + ss.append((self.timeout_index, ' %d: TIMEOUT' % + self.timeout_index)) + ss.sort() + ss = list(zip(*ss))[1] + return '\n'.join(ss) + + def search(self, buffer, freshlen, searchwindowsize=None): + '''This searches 'buffer' for the first occurrence of one of the regular + expressions. 'freshlen' must indicate the number of bytes at the end of + 'buffer' which have not been searched before. + + See class spawn for the 'searchwindowsize' argument. + + If there is a match this returns the index of that string, and sets + 'start', 'end' and 'match'. Otherwise, returns -1.''' + + first_match = None + # 'freshlen' doesn't help here -- we cannot predict the + # length of a match, and the re module provides no help. + if searchwindowsize is None: + searchstart = 0 + else: + searchstart = max(0, len(buffer) - searchwindowsize) + for index, s in self._searches: + match = s.search(buffer, searchstart) + if match is None: + continue + n = match.start() + if first_match is None or n < first_match: + first_match = n + the_match = match + best_index = index + if first_match is None: + return -1 + self.start = first_match + self.match = the_match + self.end = self.match.end() + return best_index diff --git a/lib/python3.12/site-packages/pexpect/fdpexpect.py b/lib/python3.12/site-packages/pexpect/fdpexpect.py new file mode 100644 index 0000000000000000000000000000000000000000..140bdfeeda6992acf01dae8bf2d058ac84fc8647 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/fdpexpect.py @@ -0,0 +1,152 @@ +'''This is like :mod:`pexpect`, but it will work with any file descriptor that you +pass it. You are responsible for opening and close the file descriptor. +This allows you to use Pexpect with sockets and named pipes (FIFOs). + +.. note:: + socket.fileno() does not give a readable file descriptor on windows. + Use :mod:`pexpect.socket_pexpect` for cross-platform socket support + +PEXPECT LICENSE + + This license is approved by the OSI and FSF as GPL-compatible. + http://opensource.org/licenses/isc-license.txt + + Copyright (c) 2012, Noah Spurrier + PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY + PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE + COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +''' + +from .spawnbase import SpawnBase +from .exceptions import ExceptionPexpect, TIMEOUT +from .utils import select_ignore_interrupts, poll_ignore_interrupts +import os + +__all__ = ['fdspawn'] + +class fdspawn(SpawnBase): + '''This is like pexpect.spawn but allows you to supply your own open file + descriptor. For example, you could use it to read through a file looking + for patterns, or to control a modem or serial device. ''' + + def __init__ (self, fd, args=None, timeout=30, maxread=2000, searchwindowsize=None, + logfile=None, encoding=None, codec_errors='strict', use_poll=False): + '''This takes a file descriptor (an int) or an object that support the + fileno() method (returning an int). All Python file-like objects + support fileno(). ''' + + if type(fd) != type(0) and hasattr(fd, 'fileno'): + fd = fd.fileno() + + if type(fd) != type(0): + raise ExceptionPexpect('The fd argument is not an int. If this is a command string then maybe you want to use pexpect.spawn.') + + try: # make sure fd is a valid file descriptor + os.fstat(fd) + except OSError: + raise ExceptionPexpect('The fd argument is not a valid file descriptor.') + + self.args = None + self.command = None + SpawnBase.__init__(self, timeout, maxread, searchwindowsize, logfile, + encoding=encoding, codec_errors=codec_errors) + self.child_fd = fd + self.own_fd = False + self.closed = False + self.name = '' % fd + self.use_poll = use_poll + + def close (self): + """Close the file descriptor. + + Calling this method a second time does nothing, but if the file + descriptor was closed elsewhere, :class:`OSError` will be raised. + """ + if self.child_fd == -1: + return + + self.flush() + os.close(self.child_fd) + self.child_fd = -1 + self.closed = True + + def isalive (self): + '''This checks if the file descriptor is still valid. If :func:`os.fstat` + does not raise an exception then we assume it is alive. ''' + + if self.child_fd == -1: + return False + try: + os.fstat(self.child_fd) + return True + except: + return False + + def terminate (self, force=False): # pragma: no cover + '''Deprecated and invalid. Just raises an exception.''' + raise ExceptionPexpect('This method is not valid for file descriptors.') + + # These four methods are left around for backwards compatibility, but not + # documented as part of fdpexpect. You're encouraged to use os.write + # directly. + def send(self, s): + "Write to fd, return number of bytes written" + s = self._coerce_send_string(s) + self._log(s, 'send') + + b = self._encoder.encode(s, final=False) + return os.write(self.child_fd, b) + + def sendline(self, s): + "Write to fd with trailing newline, return number of bytes written" + s = self._coerce_send_string(s) + return self.send(s + self.linesep) + + def write(self, s): + "Write to fd, return None" + self.send(s) + + def writelines(self, sequence): + "Call self.write() for each item in sequence" + for s in sequence: + self.write(s) + + def read_nonblocking(self, size=1, timeout=-1): + """ + Read from the file descriptor and return the result as a string. + + The read_nonblocking method of :class:`SpawnBase` assumes that a call + to os.read will not block (timeout parameter is ignored). This is not + the case for POSIX file-like objects such as sockets and serial ports. + + Use :func:`select.select`, timeout is implemented conditionally for + POSIX systems. + + :param int size: Read at most *size* bytes. + :param int timeout: Wait timeout seconds for file descriptor to be + ready to read. When -1 (default), use self.timeout. When 0, poll. + :return: String containing the bytes read + """ + if os.name == 'posix': + if timeout == -1: + timeout = self.timeout + rlist = [self.child_fd] + wlist = [] + xlist = [] + if self.use_poll: + rlist = poll_ignore_interrupts(rlist, timeout) + else: + rlist, wlist, xlist = select_ignore_interrupts( + rlist, wlist, xlist, timeout + ) + if self.child_fd not in rlist: + raise TIMEOUT('Timeout exceeded.') + return super(fdspawn, self).read_nonblocking(size) diff --git a/lib/python3.12/site-packages/pexpect/popen_spawn.py b/lib/python3.12/site-packages/pexpect/popen_spawn.py new file mode 100644 index 0000000000000000000000000000000000000000..e6bdf07d614a462a34cc6114e6ab40afa3562b72 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/popen_spawn.py @@ -0,0 +1,188 @@ +"""Provides an interface like pexpect.spawn interface using subprocess.Popen +""" +import os +import threading +import subprocess +import sys +import time +import signal +import shlex + +try: + from queue import Queue, Empty # Python 3 +except ImportError: + from Queue import Queue, Empty # Python 2 + +from .spawnbase import SpawnBase, PY3 +from .exceptions import EOF +from .utils import string_types + +class PopenSpawn(SpawnBase): + def __init__(self, cmd, timeout=30, maxread=2000, searchwindowsize=None, + logfile=None, cwd=None, env=None, encoding=None, + codec_errors='strict', preexec_fn=None): + super(PopenSpawn, self).__init__(timeout=timeout, maxread=maxread, + searchwindowsize=searchwindowsize, logfile=logfile, + encoding=encoding, codec_errors=codec_errors) + + # Note that `SpawnBase` initializes `self.crlf` to `\r\n` + # because the default behaviour for a PTY is to convert + # incoming LF to `\r\n` (see the `onlcr` flag and + # https://stackoverflow.com/a/35887657/5397009). Here we set + # it to `os.linesep` because that is what the spawned + # application outputs by default and `popen` doesn't translate + # anything. + if encoding is None: + self.crlf = os.linesep.encode ("ascii") + else: + self.crlf = self.string_type (os.linesep) + + kwargs = dict(bufsize=0, stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, stdout=subprocess.PIPE, + cwd=cwd, preexec_fn=preexec_fn, env=env) + + if sys.platform == 'win32': + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + kwargs['startupinfo'] = startupinfo + kwargs['creationflags'] = subprocess.CREATE_NEW_PROCESS_GROUP + + if isinstance(cmd, string_types) and sys.platform != 'win32': + cmd = shlex.split(cmd, posix=os.name == 'posix') + + self.proc = subprocess.Popen(cmd, **kwargs) + self.pid = self.proc.pid + self.closed = False + self._buf = self.string_type() + + self._read_queue = Queue() + self._read_thread = threading.Thread(target=self._read_incoming) + self._read_thread.daemon = True + self._read_thread.start() + + _read_reached_eof = False + + def read_nonblocking(self, size, timeout): + buf = self._buf + if self._read_reached_eof: + # We have already finished reading. Use up any buffered data, + # then raise EOF + if buf: + self._buf = buf[size:] + return buf[:size] + else: + self.flag_eof = True + raise EOF('End Of File (EOF).') + + if timeout == -1: + timeout = self.timeout + elif timeout is None: + timeout = 1e6 + + t0 = time.time() + while (time.time() - t0) < timeout and size and len(buf) < size: + try: + incoming = self._read_queue.get_nowait() + except Empty: + break + else: + if incoming is None: + self._read_reached_eof = True + break + + buf += self._decoder.decode(incoming, final=False) + + r, self._buf = buf[:size], buf[size:] + + self._log(r, 'read') + return r + + def _read_incoming(self): + """Run in a thread to move output from a pipe to a queue.""" + fileno = self.proc.stdout.fileno() + while 1: + buf = b'' + try: + buf = os.read(fileno, 1024) + except OSError as e: + self._log(e, 'read') + + if not buf: + # This indicates we have reached EOF + self._read_queue.put(None) + return + + self._read_queue.put(buf) + + def write(self, s): + '''This is similar to send() except that there is no return value. + ''' + self.send(s) + + def writelines(self, sequence): + '''This calls write() for each element in the sequence. + + The sequence can be any iterable object producing strings, typically a + list of strings. This does not add line separators. There is no return + value. + ''' + for s in sequence: + self.send(s) + + def send(self, s): + '''Send data to the subprocess' stdin. + + Returns the number of bytes written. + ''' + s = self._coerce_send_string(s) + self._log(s, 'send') + + b = self._encoder.encode(s, final=False) + if PY3: + return self.proc.stdin.write(b) + else: + # On Python 2, .write() returns None, so we return the length of + # bytes written ourselves. This assumes they all got written. + self.proc.stdin.write(b) + return len(b) + + def sendline(self, s=''): + '''Wraps send(), sending string ``s`` to child process, with os.linesep + automatically appended. Returns number of bytes written. ''' + + n = self.send(s) + return n + self.send(self.linesep) + + def wait(self): + '''Wait for the subprocess to finish. + + Returns the exit code. + ''' + status = self.proc.wait() + if status >= 0: + self.exitstatus = status + self.signalstatus = None + else: + self.exitstatus = None + self.signalstatus = -status + self.terminated = True + return status + + def kill(self, sig): + '''Sends a Unix signal to the subprocess. + + Use constants from the :mod:`signal` module to specify which signal. + ''' + if sys.platform == 'win32': + if sig in [signal.SIGINT, signal.CTRL_C_EVENT]: + sig = signal.CTRL_C_EVENT + elif sig in [signal.SIGBREAK, signal.CTRL_BREAK_EVENT]: + sig = signal.CTRL_BREAK_EVENT + else: + sig = signal.SIGTERM + + os.kill(self.proc.pid, sig) + + def sendeof(self): + '''Closes the stdin pipe from the writing end.''' + self.proc.stdin.close() diff --git a/lib/python3.12/site-packages/pexpect/pty_spawn.py b/lib/python3.12/site-packages/pexpect/pty_spawn.py new file mode 100644 index 0000000000000000000000000000000000000000..8e28ca7cd7de46ede116b2b9d354e50669f05de2 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/pty_spawn.py @@ -0,0 +1,860 @@ +import os +import sys +import time +import pty +import tty +import errno +import signal +from contextlib import contextmanager + +import ptyprocess +from ptyprocess.ptyprocess import use_native_pty_fork + +from .exceptions import ExceptionPexpect, EOF, TIMEOUT +from .spawnbase import SpawnBase +from .utils import ( + which, split_command_line, select_ignore_interrupts, poll_ignore_interrupts +) + +@contextmanager +def _wrap_ptyprocess_err(): + """Turn ptyprocess errors into our own ExceptionPexpect errors""" + try: + yield + except ptyprocess.PtyProcessError as e: + raise ExceptionPexpect(*e.args) + +PY3 = (sys.version_info[0] >= 3) + +class spawn(SpawnBase): + '''This is the main class interface for Pexpect. Use this class to start + and control child applications. ''' + + # This is purely informational now - changing it has no effect + use_native_pty_fork = use_native_pty_fork + + def __init__(self, command, args=[], timeout=30, maxread=2000, + searchwindowsize=None, logfile=None, cwd=None, env=None, + ignore_sighup=False, echo=True, preexec_fn=None, + encoding=None, codec_errors='strict', dimensions=None, + use_poll=False): + '''This is the constructor. The command parameter may be a string that + includes a command and any arguments to the command. For example:: + + child = pexpect.spawn('/usr/bin/ftp') + child = pexpect.spawn('/usr/bin/ssh user@example.com') + child = pexpect.spawn('ls -latr /tmp') + + You may also construct it with a list of arguments like so:: + + child = pexpect.spawn('/usr/bin/ftp', []) + child = pexpect.spawn('/usr/bin/ssh', ['user@example.com']) + child = pexpect.spawn('ls', ['-latr', '/tmp']) + + After this the child application will be created and will be ready to + talk to. For normal use, see expect() and send() and sendline(). + + Remember that Pexpect does NOT interpret shell meta characters such as + redirect, pipe, or wild cards (``>``, ``|``, or ``*``). This is a + common mistake. If you want to run a command and pipe it through + another command then you must also start a shell. For example:: + + child = pexpect.spawn('/bin/bash -c "ls -l | grep LOG > logs.txt"') + child.expect(pexpect.EOF) + + The second form of spawn (where you pass a list of arguments) is useful + in situations where you wish to spawn a command and pass it its own + argument list. This can make syntax more clear. For example, the + following is equivalent to the previous example:: + + shell_cmd = 'ls -l | grep LOG > logs.txt' + child = pexpect.spawn('/bin/bash', ['-c', shell_cmd]) + child.expect(pexpect.EOF) + + The maxread attribute sets the read buffer size. This is maximum number + of bytes that Pexpect will try to read from a TTY at one time. Setting + the maxread size to 1 will turn off buffering. Setting the maxread + value higher may help performance in cases where large amounts of + output are read back from the child. This feature is useful in + conjunction with searchwindowsize. + + When the keyword argument *searchwindowsize* is None (default), the + full buffer is searched at each iteration of receiving incoming data. + The default number of bytes scanned at each iteration is very large + and may be reduced to collaterally reduce search cost. After + :meth:`~.expect` returns, the full buffer attribute remains up to + size *maxread* irrespective of *searchwindowsize* value. + + When the keyword argument ``timeout`` is specified as a number, + (default: *30*), then :class:`TIMEOUT` will be raised after the value + specified has elapsed, in seconds, for any of the :meth:`~.expect` + family of method calls. When None, TIMEOUT will not be raised, and + :meth:`~.expect` may block indefinitely until match. + + + The logfile member turns on or off logging. All input and output will + be copied to the given file object. Set logfile to None to stop + logging. This is the default. Set logfile to sys.stdout to echo + everything to standard output. The logfile is flushed after each write. + + Example log input and output to a file:: + + child = pexpect.spawn('some_command') + fout = open('mylog.txt','wb') + child.logfile = fout + + Example log to stdout:: + + # In Python 2: + child = pexpect.spawn('some_command') + child.logfile = sys.stdout + + # In Python 3, we'll use the ``encoding`` argument to decode data + # from the subprocess and handle it as unicode: + child = pexpect.spawn('some_command', encoding='utf-8') + child.logfile = sys.stdout + + The logfile_read and logfile_send members can be used to separately log + the input from the child and output sent to the child. Sometimes you + don't want to see everything you write to the child. You only want to + log what the child sends back. For example:: + + child = pexpect.spawn('some_command') + child.logfile_read = sys.stdout + + You will need to pass an encoding to spawn in the above code if you are + using Python 3. + + To separately log output sent to the child use logfile_send:: + + child.logfile_send = fout + + If ``ignore_sighup`` is True, the child process will ignore SIGHUP + signals. The default is False from Pexpect 4.0, meaning that SIGHUP + will be handled normally by the child. + + The delaybeforesend helps overcome a weird behavior that many users + were experiencing. The typical problem was that a user would expect() a + "Password:" prompt and then immediately call sendline() to send the + password. The user would then see that their password was echoed back + to them. Passwords don't normally echo. The problem is caused by the + fact that most applications print out the "Password" prompt and then + turn off stdin echo, but if you send your password before the + application turned off echo, then you get your password echoed. + Normally this wouldn't be a problem when interacting with a human at a + real keyboard. If you introduce a slight delay just before writing then + this seems to clear up the problem. This was such a common problem for + many users that I decided that the default pexpect behavior should be + to sleep just before writing to the child application. 1/20th of a + second (50 ms) seems to be enough to clear up the problem. You can set + delaybeforesend to None to return to the old behavior. + + Note that spawn is clever about finding commands on your path. + It uses the same logic that "which" uses to find executables. + + If you wish to get the exit status of the child you must call the + close() method. The exit or signal status of the child will be stored + in self.exitstatus or self.signalstatus. If the child exited normally + then exitstatus will store the exit return code and signalstatus will + be None. If the child was terminated abnormally with a signal then + signalstatus will store the signal value and exitstatus will be None:: + + child = pexpect.spawn('some_command') + child.close() + print(child.exitstatus, child.signalstatus) + + If you need more detail you can also read the self.status member which + stores the status returned by os.waitpid. You can interpret this using + os.WIFEXITED/os.WEXITSTATUS or os.WIFSIGNALED/os.TERMSIG. + + The echo attribute may be set to False to disable echoing of input. + As a pseudo-terminal, all input echoed by the "keyboard" (send() + or sendline()) will be repeated to output. For many cases, it is + not desirable to have echo enabled, and it may be later disabled + using setecho(False) followed by waitnoecho(). However, for some + platforms such as Solaris, this is not possible, and should be + disabled immediately on spawn. + + If preexec_fn is given, it will be called in the child process before + launching the given command. This is useful to e.g. reset inherited + signal handlers. + + The dimensions attribute specifies the size of the pseudo-terminal as + seen by the subprocess, and is specified as a two-entry tuple (rows, + columns). If this is unspecified, the defaults in ptyprocess will apply. + + The use_poll attribute enables using select.poll() over select.select() + for socket handling. This is handy if your system could have > 1024 fds + ''' + super(spawn, self).__init__(timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, + logfile=logfile, encoding=encoding, codec_errors=codec_errors) + self.STDIN_FILENO = pty.STDIN_FILENO + self.STDOUT_FILENO = pty.STDOUT_FILENO + self.STDERR_FILENO = pty.STDERR_FILENO + self.str_last_chars = 100 + self.cwd = cwd + self.env = env + self.echo = echo + self.ignore_sighup = ignore_sighup + self.__irix_hack = sys.platform.lower().startswith('irix') + if command is None: + self.command = None + self.args = None + self.name = '' + else: + self._spawn(command, args, preexec_fn, dimensions) + self.use_poll = use_poll + + def __str__(self): + '''This returns a human-readable string that represents the state of + the object. ''' + + s = [] + s.append(repr(self)) + s.append('command: ' + str(self.command)) + s.append('args: %r' % (self.args,)) + s.append('buffer (last %s chars): %r' % (self.str_last_chars,self.buffer[-self.str_last_chars:])) + s.append('before (last %s chars): %r' % (self.str_last_chars,self.before[-self.str_last_chars:] if self.before else '')) + s.append('after: %r' % (self.after,)) + s.append('match: %r' % (self.match,)) + s.append('match_index: ' + str(self.match_index)) + s.append('exitstatus: ' + str(self.exitstatus)) + if hasattr(self, 'ptyproc'): + s.append('flag_eof: ' + str(self.flag_eof)) + s.append('pid: ' + str(self.pid)) + s.append('child_fd: ' + str(self.child_fd)) + s.append('closed: ' + str(self.closed)) + s.append('timeout: ' + str(self.timeout)) + s.append('delimiter: ' + str(self.delimiter)) + s.append('logfile: ' + str(self.logfile)) + s.append('logfile_read: ' + str(self.logfile_read)) + s.append('logfile_send: ' + str(self.logfile_send)) + s.append('maxread: ' + str(self.maxread)) + s.append('ignorecase: ' + str(self.ignorecase)) + s.append('searchwindowsize: ' + str(self.searchwindowsize)) + s.append('delaybeforesend: ' + str(self.delaybeforesend)) + s.append('delayafterclose: ' + str(self.delayafterclose)) + s.append('delayafterterminate: ' + str(self.delayafterterminate)) + return '\n'.join(s) + + def _spawn(self, command, args=[], preexec_fn=None, dimensions=None): + '''This starts the given command in a child process. This does all the + fork/exec type of stuff for a pty. This is called by __init__. If args + is empty then command will be parsed (split on spaces) and args will be + set to parsed arguments. ''' + + # The pid and child_fd of this object get set by this method. + # Note that it is difficult for this method to fail. + # You cannot detect if the child process cannot start. + # So the only way you can tell if the child process started + # or not is to try to read from the file descriptor. If you get + # EOF immediately then it means that the child is already dead. + # That may not necessarily be bad because you may have spawned a child + # that performs some task; creates no stdout output; and then dies. + + # If command is an int type then it may represent a file descriptor. + if isinstance(command, type(0)): + raise ExceptionPexpect('Command is an int type. ' + + 'If this is a file descriptor then maybe you want to ' + + 'use fdpexpect.fdspawn which takes an existing ' + + 'file descriptor instead of a command string.') + + if not isinstance(args, type([])): + raise TypeError('The argument, args, must be a list.') + + if args == []: + self.args = split_command_line(command) + self.command = self.args[0] + else: + # Make a shallow copy of the args list. + self.args = args[:] + self.args.insert(0, command) + self.command = command + + command_with_path = which(self.command, env=self.env) + if command_with_path is None: + raise ExceptionPexpect('The command was not found or was not ' + + 'executable: %s.' % self.command) + self.command = command_with_path + self.args[0] = self.command + + self.name = '<' + ' '.join(self.args) + '>' + + assert self.pid is None, 'The pid member must be None.' + assert self.command is not None, 'The command member must not be None.' + + kwargs = {'echo': self.echo, 'preexec_fn': preexec_fn} + if self.ignore_sighup: + def preexec_wrapper(): + "Set SIGHUP to be ignored, then call the real preexec_fn" + signal.signal(signal.SIGHUP, signal.SIG_IGN) + if preexec_fn is not None: + preexec_fn() + kwargs['preexec_fn'] = preexec_wrapper + + if dimensions is not None: + kwargs['dimensions'] = dimensions + + if self.encoding is not None: + # Encode command line using the specified encoding + self.args = [a if isinstance(a, bytes) else a.encode(self.encoding) + for a in self.args] + + self.ptyproc = self._spawnpty(self.args, env=self.env, + cwd=self.cwd, **kwargs) + + self.pid = self.ptyproc.pid + self.child_fd = self.ptyproc.fd + + + self.terminated = False + self.closed = False + + def _spawnpty(self, args, **kwargs): + '''Spawn a pty and return an instance of PtyProcess.''' + return ptyprocess.PtyProcess.spawn(args, **kwargs) + + def close(self, force=True): + '''This closes the connection with the child application. Note that + calling close() more than once is valid. This emulates standard Python + behavior with files. Set force to True if you want to make sure that + the child is terminated (SIGKILL is sent if the child ignores SIGHUP + and SIGINT). ''' + + self.flush() + with _wrap_ptyprocess_err(): + # PtyProcessError may be raised if it is not possible to terminate + # the child. + self.ptyproc.close(force=force) + self.isalive() # Update exit status from ptyproc + self.child_fd = -1 + self.closed = True + + def isatty(self): + '''This returns True if the file descriptor is open and connected to a + tty(-like) device, else False. + + On SVR4-style platforms implementing streams, such as SunOS and HP-UX, + the child pty may not appear as a terminal device. This means + methods such as setecho(), setwinsize(), getwinsize() may raise an + IOError. ''' + + return os.isatty(self.child_fd) + + def waitnoecho(self, timeout=-1): + '''This waits until the terminal ECHO flag is set False. This returns + True if the echo mode is off. This returns False if the ECHO flag was + not set False before the timeout. This can be used to detect when the + child is waiting for a password. Usually a child application will turn + off echo mode when it is waiting for the user to enter a password. For + example, instead of expecting the "password:" prompt you can wait for + the child to set ECHO off:: + + p = pexpect.spawn('ssh user@example.com') + p.waitnoecho() + p.sendline(mypassword) + + If timeout==-1 then this method will use the value in self.timeout. + If timeout==None then this method to block until ECHO flag is False. + ''' + + if timeout == -1: + timeout = self.timeout + if timeout is not None: + end_time = time.time() + timeout + while True: + if not self.getecho(): + return True + if timeout < 0 and timeout is not None: + return False + if timeout is not None: + timeout = end_time - time.time() + time.sleep(0.1) + + def getecho(self): + '''This returns the terminal echo mode. This returns True if echo is + on or False if echo is off. Child applications that are expecting you + to enter a password often set ECHO False. See waitnoecho(). + + Not supported on platforms where ``isatty()`` returns False. ''' + return self.ptyproc.getecho() + + def setecho(self, state): + '''This sets the terminal echo mode on or off. Note that anything the + child sent before the echo will be lost, so you should be sure that + your input buffer is empty before you call setecho(). For example, the + following will work as expected:: + + p = pexpect.spawn('cat') # Echo is on by default. + p.sendline('1234') # We expect see this twice from the child... + p.expect(['1234']) # ... once from the tty echo... + p.expect(['1234']) # ... and again from cat itself. + p.setecho(False) # Turn off tty echo + p.sendline('abcd') # We will set this only once (echoed by cat). + p.sendline('wxyz') # We will set this only once (echoed by cat) + p.expect(['abcd']) + p.expect(['wxyz']) + + The following WILL NOT WORK because the lines sent before the setecho + will be lost:: + + p = pexpect.spawn('cat') + p.sendline('1234') + p.setecho(False) # Turn off tty echo + p.sendline('abcd') # We will set this only once (echoed by cat). + p.sendline('wxyz') # We will set this only once (echoed by cat) + p.expect(['1234']) + p.expect(['1234']) + p.expect(['abcd']) + p.expect(['wxyz']) + + + Not supported on platforms where ``isatty()`` returns False. + ''' + return self.ptyproc.setecho(state) + + def read_nonblocking(self, size=1, timeout=-1): + '''This reads at most size characters from the child application. It + includes a timeout. If the read does not complete within the timeout + period then a TIMEOUT exception is raised. If the end of file is read + then an EOF exception will be raised. If a logfile is specified, a + copy is written to that log. + + If timeout is None then the read may block indefinitely. + If timeout is -1 then the self.timeout value is used. If timeout is 0 + then the child is polled and if there is no data immediately ready + then this will raise a TIMEOUT exception. + + The timeout refers only to the amount of time to read at least one + character. This is not affected by the 'size' parameter, so if you call + read_nonblocking(size=100, timeout=30) and only one character is + available right away then one character will be returned immediately. + It will not wait for 30 seconds for another 99 characters to come in. + + On the other hand, if there are bytes available to read immediately, + all those bytes will be read (up to the buffer size). So, if the + buffer size is 1 megabyte and there is 1 megabyte of data available + to read, the buffer will be filled, regardless of timeout. + + This is a wrapper around os.read(). It uses select.select() or + select.poll() to implement the timeout. ''' + + if self.closed: + raise ValueError('I/O operation on closed file.') + + if self.use_poll: + def select(timeout): + return poll_ignore_interrupts([self.child_fd], timeout) + else: + def select(timeout): + return select_ignore_interrupts([self.child_fd], [], [], timeout)[0] + + # If there is data available to read right now, read as much as + # we can. We do this to increase performance if there are a lot + # of bytes to be read. This also avoids calling isalive() too + # often. See also: + # * https://github.com/pexpect/pexpect/pull/304 + # * http://trac.sagemath.org/ticket/10295 + if select(0): + try: + incoming = super(spawn, self).read_nonblocking(size) + except EOF: + # Maybe the child is dead: update some attributes in that case + self.isalive() + raise + while len(incoming) < size and select(0): + try: + incoming += super(spawn, self).read_nonblocking(size - len(incoming)) + except EOF: + # Maybe the child is dead: update some attributes in that case + self.isalive() + # Don't raise EOF, just return what we read so far. + return incoming + return incoming + + if timeout == -1: + timeout = self.timeout + + if not self.isalive(): + # The process is dead, but there may or may not be data + # available to read. Note that some systems such as Solaris + # do not give an EOF when the child dies. In fact, you can + # still try to read from the child_fd -- it will block + # forever or until TIMEOUT. For that reason, it's important + # to do this check before calling select() with timeout. + if select(0): + return super(spawn, self).read_nonblocking(size) + self.flag_eof = True + raise EOF('End Of File (EOF). Braindead platform.') + elif self.__irix_hack: + # Irix takes a long time before it realizes a child was terminated. + # Make sure that the timeout is at least 2 seconds. + # FIXME So does this mean Irix systems are forced to always have + # FIXME a 2 second delay when calling read_nonblocking? That sucks. + if timeout is not None and timeout < 2: + timeout = 2 + + # Because of the select(0) check above, we know that no data + # is available right now. But if a non-zero timeout is given + # (possibly timeout=None), we call select() with a timeout. + if (timeout != 0) and select(timeout): + return super(spawn, self).read_nonblocking(size) + + if not self.isalive(): + # Some platforms, such as Irix, will claim that their + # processes are alive; timeout on the select; and + # then finally admit that they are not alive. + self.flag_eof = True + raise EOF('End of File (EOF). Very slow platform.') + else: + raise TIMEOUT('Timeout exceeded.') + + def write(self, s): + '''This is similar to send() except that there is no return value. + ''' + + self.send(s) + + def writelines(self, sequence): + '''This calls write() for each element in the sequence. The sequence + can be any iterable object producing strings, typically a list of + strings. This does not add line separators. There is no return value. + ''' + + for s in sequence: + self.write(s) + + def send(self, s): + '''Sends string ``s`` to the child process, returning the number of + bytes written. If a logfile is specified, a copy is written to that + log. + + The default terminal input mode is canonical processing unless set + otherwise by the child process. This allows backspace and other line + processing to be performed prior to transmitting to the receiving + program. As this is buffered, there is a limited size of such buffer. + + On Linux systems, this is 4096 (defined by N_TTY_BUF_SIZE). All + other systems honor the POSIX.1 definition PC_MAX_CANON -- 1024 + on OSX, 256 on OpenSolaris, and 1920 on FreeBSD. + + This value may be discovered using fpathconf(3):: + + >>> from os import fpathconf + >>> print(fpathconf(0, 'PC_MAX_CANON')) + 256 + + On such a system, only 256 bytes may be received per line. Any + subsequent bytes received will be discarded. BEL (``'\a'``) is then + sent to output if IMAXBEL (termios.h) is set by the tty driver. + This is usually enabled by default. Linux does not honor this as + an option -- it behaves as though it is always set on. + + Canonical input processing may be disabled altogether by executing + a shell, then stty(1), before executing the final program:: + + >>> bash = pexpect.spawn('/bin/bash', echo=False) + >>> bash.sendline('stty -icanon') + >>> bash.sendline('base64') + >>> bash.sendline('x' * 5000) + ''' + + if self.delaybeforesend is not None: + time.sleep(self.delaybeforesend) + + s = self._coerce_send_string(s) + self._log(s, 'send') + + b = self._encoder.encode(s, final=False) + return os.write(self.child_fd, b) + + def sendline(self, s=''): + '''Wraps send(), sending string ``s`` to child process, with + ``os.linesep`` automatically appended. Returns number of bytes + written. Only a limited number of bytes may be sent for each + line in the default terminal mode, see docstring of :meth:`send`. + ''' + s = self._coerce_send_string(s) + return self.send(s + self.linesep) + + def _log_control(self, s): + """Write control characters to the appropriate log files""" + if self.encoding is not None: + s = s.decode(self.encoding, 'replace') + self._log(s, 'send') + + def sendcontrol(self, char): + '''Helper method that wraps send() with mnemonic access for sending control + character to the child (such as Ctrl-C or Ctrl-D). For example, to send + Ctrl-G (ASCII 7, bell, '\a'):: + + child.sendcontrol('g') + + See also, sendintr() and sendeof(). + ''' + n, byte = self.ptyproc.sendcontrol(char) + self._log_control(byte) + return n + + def sendeof(self): + '''This sends an EOF to the child. This sends a character which causes + the pending parent output buffer to be sent to the waiting child + program without waiting for end-of-line. If it is the first character + of the line, the read() in the user program returns 0, which signifies + end-of-file. This means to work as expected a sendeof() has to be + called at the beginning of a line. This method does not send a newline. + It is the responsibility of the caller to ensure the eof is sent at the + beginning of a line. ''' + + n, byte = self.ptyproc.sendeof() + self._log_control(byte) + + def sendintr(self): + '''This sends a SIGINT to the child. It does not require + the SIGINT to be the first character on a line. ''' + + n, byte = self.ptyproc.sendintr() + self._log_control(byte) + + @property + def flag_eof(self): + return self.ptyproc.flag_eof + + @flag_eof.setter + def flag_eof(self, value): + self.ptyproc.flag_eof = value + + def eof(self): + '''This returns True if the EOF exception was ever raised. + ''' + return self.flag_eof + + def terminate(self, force=False): + '''This forces a child process to terminate. It starts nicely with + SIGHUP and SIGINT. If "force" is True then moves onto SIGKILL. This + returns True if the child was terminated. This returns False if the + child could not be terminated. ''' + + if not self.isalive(): + return True + try: + self.kill(signal.SIGHUP) + time.sleep(self.delayafterterminate) + if not self.isalive(): + return True + self.kill(signal.SIGCONT) + time.sleep(self.delayafterterminate) + if not self.isalive(): + return True + self.kill(signal.SIGINT) + time.sleep(self.delayafterterminate) + if not self.isalive(): + return True + if force: + self.kill(signal.SIGKILL) + time.sleep(self.delayafterterminate) + if not self.isalive(): + return True + else: + return False + return False + except OSError: + # I think there are kernel timing issues that sometimes cause + # this to happen. I think isalive() reports True, but the + # process is dead to the kernel. + # Make one last attempt to see if the kernel is up to date. + time.sleep(self.delayafterterminate) + if not self.isalive(): + return True + else: + return False + + def wait(self): + '''This waits until the child exits. This is a blocking call. This will + not read any data from the child, so this will block forever if the + child has unread output and has terminated. In other words, the child + may have printed output then called exit(), but, the child is + technically still alive until its output is read by the parent. + + This method is non-blocking if :meth:`wait` has already been called + previously or :meth:`isalive` method returns False. It simply returns + the previously determined exit status. + ''' + + ptyproc = self.ptyproc + with _wrap_ptyprocess_err(): + # exception may occur if "Is some other process attempting + # "job control with our child pid?" + exitstatus = ptyproc.wait() + self.status = ptyproc.status + self.exitstatus = ptyproc.exitstatus + self.signalstatus = ptyproc.signalstatus + self.terminated = True + + return exitstatus + + def isalive(self): + '''This tests if the child process is running or not. This is + non-blocking. If the child was terminated then this will read the + exitstatus or signalstatus of the child. This returns True if the child + process appears to be running or False if not. It can take literally + SECONDS for Solaris to return the right status. ''' + + ptyproc = self.ptyproc + with _wrap_ptyprocess_err(): + alive = ptyproc.isalive() + + if not alive: + self.status = ptyproc.status + self.exitstatus = ptyproc.exitstatus + self.signalstatus = ptyproc.signalstatus + self.terminated = True + + return alive + + def kill(self, sig): + + '''This sends the given signal to the child application. In keeping + with UNIX tradition it has a misleading name. It does not necessarily + kill the child unless you send the right signal. ''' + + # Same as os.kill, but the pid is given for you. + if self.isalive(): + os.kill(self.pid, sig) + + def getwinsize(self): + '''This returns the terminal window size of the child tty. The return + value is a tuple of (rows, cols). ''' + return self.ptyproc.getwinsize() + + def setwinsize(self, rows, cols): + '''This sets the terminal window size of the child tty. This will cause + a SIGWINCH signal to be sent to the child. This does not change the + physical window size. It changes the size reported to TTY-aware + applications like vi or curses -- applications that respond to the + SIGWINCH signal. ''' + return self.ptyproc.setwinsize(rows, cols) + + + def interact(self, escape_character=chr(29), + input_filter=None, output_filter=None): + + '''This gives control of the child process to the interactive user (the + human at the keyboard). Keystrokes are sent to the child process, and + the stdout and stderr output of the child process is printed. This + simply echos the child stdout and child stderr to the real stdout and + it echos the real stdin to the child stdin. When the user types the + escape_character this method will return None. The escape_character + will not be transmitted. The default for escape_character is + entered as ``Ctrl - ]``, the very same as BSD telnet. To prevent + escaping, escape_character may be set to None. + + If a logfile is specified, then the data sent and received from the + child process in interact mode is duplicated to the given log. + + You may pass in optional input and output filter functions. These + functions should take bytes array and return bytes array too. Even + with ``encoding='utf-8'`` support, meth:`interact` will always pass + input_filter and output_filter bytes. You may need to wrap your + function to decode and encode back to UTF-8. + + The output_filter will be passed all the output from the child process. + The input_filter will be passed all the keyboard input from the user. + The input_filter is run BEFORE the check for the escape_character. + + Note that if you change the window size of the parent the SIGWINCH + signal will not be passed through to the child. If you want the child + window size to change when the parent's window size changes then do + something like the following example:: + + import pexpect, struct, fcntl, termios, signal, sys + def sigwinch_passthrough (sig, data): + s = struct.pack("HHHH", 0, 0, 0, 0) + a = struct.unpack('hhhh', fcntl.ioctl(sys.stdout.fileno(), + termios.TIOCGWINSZ , s)) + if not p.closed: + p.setwinsize(a[0],a[1]) + + # Note this 'p' is global and used in sigwinch_passthrough. + p = pexpect.spawn('/bin/bash') + signal.signal(signal.SIGWINCH, sigwinch_passthrough) + p.interact() + ''' + + # Flush the buffer. + self.write_to_stdout(self.buffer) + self.stdout.flush() + self._buffer = self.buffer_type() + mode = tty.tcgetattr(self.STDIN_FILENO) + tty.setraw(self.STDIN_FILENO) + if escape_character is not None and PY3: + escape_character = escape_character.encode('latin-1') + try: + self.__interact_copy(escape_character, input_filter, output_filter) + finally: + tty.tcsetattr(self.STDIN_FILENO, tty.TCSAFLUSH, mode) + + def __interact_writen(self, fd, data): + '''This is used by the interact() method. + ''' + + while data != b'' and self.isalive(): + n = os.write(fd, data) + data = data[n:] + + def __interact_read(self, fd): + '''This is used by the interact() method. + ''' + + return os.read(fd, 1000) + + def __interact_copy( + self, escape_character=None, input_filter=None, output_filter=None + ): + + '''This is used by the interact() method. + ''' + + while self.isalive(): + if self.use_poll: + r = poll_ignore_interrupts([self.child_fd, self.STDIN_FILENO]) + else: + r, w, e = select_ignore_interrupts( + [self.child_fd, self.STDIN_FILENO], [], [] + ) + if self.child_fd in r: + try: + data = self.__interact_read(self.child_fd) + except OSError as err: + if err.args[0] == errno.EIO: + # Linux-style EOF + break + raise + if data == b'': + # BSD-style EOF + break + if output_filter: + data = output_filter(data) + self._log(data, 'read') + os.write(self.STDOUT_FILENO, data) + if self.STDIN_FILENO in r: + data = self.__interact_read(self.STDIN_FILENO) + if input_filter: + data = input_filter(data) + i = -1 + if escape_character is not None: + i = data.rfind(escape_character) + if i != -1: + data = data[:i] + if data: + self._log(data, 'send') + self.__interact_writen(self.child_fd, data) + break + self._log(data, 'send') + self.__interact_writen(self.child_fd, data) + + +def spawnu(*args, **kwargs): + """Deprecated: pass encoding to spawn() instead.""" + kwargs.setdefault('encoding', 'utf-8') + return spawn(*args, **kwargs) diff --git a/lib/python3.12/site-packages/pexpect/pxssh.py b/lib/python3.12/site-packages/pexpect/pxssh.py new file mode 100644 index 0000000000000000000000000000000000000000..742f59e4069e7f171801ba4a01373aa938f13df5 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/pxssh.py @@ -0,0 +1,540 @@ +'''This class extends pexpect.spawn to specialize setting up SSH connections. +This adds methods for login, logout, and expecting the shell prompt. + +PEXPECT LICENSE + + This license is approved by the OSI and FSF as GPL-compatible. + http://opensource.org/licenses/isc-license.txt + + Copyright (c) 2012, Noah Spurrier + PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY + PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE + COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +''' + +from pexpect import ExceptionPexpect, TIMEOUT, EOF, spawn +import time +import os +import sys +import re + +__all__ = ['ExceptionPxssh', 'pxssh'] + +# Exception classes used by this module. +class ExceptionPxssh(ExceptionPexpect): + '''Raised for pxssh exceptions. + ''' + +if sys.version_info > (3, 0): + from shlex import quote +else: + _find_unsafe = re.compile(r'[^\w@%+=:,./-]').search + + def quote(s): + """Return a shell-escaped version of the string *s*.""" + if not s: + return "''" + if _find_unsafe(s) is None: + return s + + # use single quotes, and put single quotes into double quotes + # the string $'b is then quoted as '$'"'"'b' + return "'" + s.replace("'", "'\"'\"'") + "'" + +class pxssh (spawn): + '''This class extends pexpect.spawn to specialize setting up SSH + connections. This adds methods for login, logout, and expecting the shell + prompt. It does various tricky things to handle many situations in the SSH + login process. For example, if the session is your first login, then pxssh + automatically accepts the remote certificate; or if you have public key + authentication setup then pxssh won't wait for the password prompt. + + pxssh uses the shell prompt to synchronize output from the remote host. In + order to make this more robust it sets the shell prompt to something more + unique than just $ or #. This should work on most Borne/Bash or Csh style + shells. + + Example that runs a few commands on a remote server and prints the result:: + + from pexpect import pxssh + import getpass + try: + s = pxssh.pxssh() + hostname = raw_input('hostname: ') + username = raw_input('username: ') + password = getpass.getpass('password: ') + s.login(hostname, username, password) + s.sendline('uptime') # run a command + s.prompt() # match the prompt + print(s.before) # print everything before the prompt. + s.sendline('ls -l') + s.prompt() + print(s.before) + s.sendline('df') + s.prompt() + print(s.before) + s.logout() + except pxssh.ExceptionPxssh as e: + print("pxssh failed on login.") + print(e) + + Example showing how to specify SSH options:: + + from pexpect import pxssh + s = pxssh.pxssh(options={ + "StrictHostKeyChecking": "no", + "UserKnownHostsFile": "/dev/null"}) + ... + + Note that if you have ssh-agent running while doing development with pxssh + then this can lead to a lot of confusion. Many X display managers (xdm, + gdm, kdm, etc.) will automatically start a GUI agent. You may see a GUI + dialog box popup asking for a password during development. You should turn + off any key agents during testing. The 'force_password' attribute will turn + off public key authentication. This will only work if the remote SSH server + is configured to allow password logins. Example of using 'force_password' + attribute:: + + s = pxssh.pxssh() + s.force_password = True + hostname = raw_input('hostname: ') + username = raw_input('username: ') + password = getpass.getpass('password: ') + s.login (hostname, username, password) + + `debug_command_string` is only for the test suite to confirm that the string + generated for SSH is correct, using this will not allow you to do + anything other than get a string back from `pxssh.pxssh.login()`. + ''' + + def __init__ (self, timeout=30, maxread=2000, searchwindowsize=None, + logfile=None, cwd=None, env=None, ignore_sighup=True, echo=True, + options={}, encoding=None, codec_errors='strict', + debug_command_string=False, use_poll=False): + + spawn.__init__(self, None, timeout=timeout, maxread=maxread, + searchwindowsize=searchwindowsize, logfile=logfile, + cwd=cwd, env=env, ignore_sighup=ignore_sighup, echo=echo, + encoding=encoding, codec_errors=codec_errors, use_poll=use_poll) + + self.name = '' + + #SUBTLE HACK ALERT! Note that the command that SETS the prompt uses a + #slightly different string than the regular expression to match it. This + #is because when you set the prompt the command will echo back, but we + #don't want to match the echoed command. So if we make the set command + #slightly different than the regex we eliminate the problem. To make the + #set command different we add a backslash in front of $. The $ doesn't + #need to be escaped, but it doesn't hurt and serves to make the set + #prompt command different than the regex. + + # used to match the command-line prompt + self.UNIQUE_PROMPT = r"\[PEXPECT\][\$\#] " + self.PROMPT = self.UNIQUE_PROMPT + + # used to set shell command-line prompt to UNIQUE_PROMPT. + self.PROMPT_SET_SH = r"PS1='[PEXPECT]\$ '" + self.PROMPT_SET_CSH = r"set prompt='[PEXPECT]\$ '" + self.PROMPT_SET_ZSH = "prompt restore;\nPS1='[PEXPECT]%(!.#.$) '" + self.SSH_OPTS = (" -o 'PubkeyAuthentication=no'") +# Disabling host key checking, makes you vulnerable to MITM attacks. +# + " -o 'StrictHostKeyChecking=no'" +# + " -o 'UserKnownHostsFile /dev/null' ") + # Disabling X11 forwarding gets rid of the annoying SSH_ASKPASS from + # displaying a GUI password dialog. I have not figured out how to + # disable only SSH_ASKPASS without also disabling X11 forwarding. + # Unsetting SSH_ASKPASS on the remote side doesn't disable it! Annoying! + #self.SSH_OPTS = "-x -o 'PubkeyAuthentication=no'" + self.force_password = False + + self.debug_command_string = debug_command_string + + # User defined SSH options, eg, + # ssh.otions = dict(StrictHostKeyChecking="no",UserKnownHostsFile="/dev/null") + self.options = options + + def levenshtein_distance(self, a, b): + '''This calculates the Levenshtein distance between a and b. + ''' + + n, m = len(a), len(b) + if n > m: + a,b = b,a + n,m = m,n + current = range(n+1) + for i in range(1,m+1): + previous, current = current, [i]+[0]*n + for j in range(1,n+1): + add, delete = previous[j]+1, current[j-1]+1 + change = previous[j-1] + if a[j-1] != b[i-1]: + change = change + 1 + current[j] = min(add, delete, change) + return current[n] + + def try_read_prompt(self, timeout_multiplier): + '''This facilitates using communication timeouts to perform + synchronization as quickly as possible, while supporting high latency + connections with a tunable worst case performance. Fast connections + should be read almost immediately. Worst case performance for this + method is timeout_multiplier * 3 seconds. + ''' + + # maximum time allowed to read the first response + first_char_timeout = timeout_multiplier * 0.5 + + # maximum time allowed between subsequent characters + inter_char_timeout = timeout_multiplier * 0.1 + + # maximum time for reading the entire prompt + total_timeout = timeout_multiplier * 3.0 + + prompt = self.string_type() + begin = time.time() + expired = 0.0 + timeout = first_char_timeout + + while expired < total_timeout: + try: + prompt += self.read_nonblocking(size=1, timeout=timeout) + expired = time.time() - begin # updated total time expired + timeout = inter_char_timeout + except TIMEOUT: + break + + return prompt + + def sync_original_prompt (self, sync_multiplier=1.0): + '''This attempts to find the prompt. Basically, press enter and record + the response; press enter again and record the response; if the two + responses are similar then assume we are at the original prompt. + This can be a slow function. Worst case with the default sync_multiplier + can take 12 seconds. Low latency connections are more likely to fail + with a low sync_multiplier. Best case sync time gets worse with a + high sync multiplier (500 ms with default). ''' + + # All of these timing pace values are magic. + # I came up with these based on what seemed reliable for + # connecting to a heavily loaded machine I have. + self.sendline() + time.sleep(0.1) + + try: + # Clear the buffer before getting the prompt. + self.try_read_prompt(sync_multiplier) + except TIMEOUT: + pass + + self.sendline() + x = self.try_read_prompt(sync_multiplier) + + self.sendline() + a = self.try_read_prompt(sync_multiplier) + + self.sendline() + b = self.try_read_prompt(sync_multiplier) + + ld = self.levenshtein_distance(a,b) + len_a = len(a) + if len_a == 0: + return False + if float(ld)/len_a < 0.4: + return True + return False + + ### TODO: This is getting messy and I'm pretty sure this isn't perfect. + ### TODO: I need to draw a flow chart for this. + ### TODO: Unit tests for SSH tunnels, remote SSH command exec, disabling original prompt sync + def login (self, server, username=None, password='', terminal_type='ansi', + original_prompt=r"[#$]", login_timeout=10, port=None, + auto_prompt_reset=True, ssh_key=None, quiet=True, + sync_multiplier=1, check_local_ip=True, + password_regex=r'(?i)(?:password:)|(?:passphrase for key)', + ssh_tunnels={}, spawn_local_ssh=True, + sync_original_prompt=True, ssh_config=None, cmd='ssh'): + '''This logs the user into the given server. + + It uses 'original_prompt' to try to find the prompt right after login. + When it finds the prompt it immediately tries to reset the prompt to + something more easily matched. The default 'original_prompt' is very + optimistic and is easily fooled. It's more reliable to try to match the original + prompt as exactly as possible to prevent false matches by server + strings such as the "Message Of The Day". On many systems you can + disable the MOTD on the remote server by creating a zero-length file + called :file:`~/.hushlogin` on the remote server. If a prompt cannot be found + then this will not necessarily cause the login to fail. In the case of + a timeout when looking for the prompt we assume that the original + prompt was so weird that we could not match it, so we use a few tricks + to guess when we have reached the prompt. Then we hope for the best and + blindly try to reset the prompt to something more unique. If that fails + then login() raises an :class:`ExceptionPxssh` exception. + + In some situations it is not possible or desirable to reset the + original prompt. In this case, pass ``auto_prompt_reset=False`` to + inhibit setting the prompt to the UNIQUE_PROMPT. Remember that pxssh + uses a unique prompt in the :meth:`prompt` method. If the original prompt is + not reset then this will disable the :meth:`prompt` method unless you + manually set the :attr:`PROMPT` attribute. + + Set ``password_regex`` if there is a MOTD message with `password` in it. + Changing this is like playing in traffic, don't (p)expect it to match straight + away. + + If you require to connect to another SSH server from the your original SSH + connection set ``spawn_local_ssh`` to `False` and this will use your current + session to do so. Setting this option to `False` and not having an active session + will trigger an error. + + Set ``ssh_key`` to a file path to an SSH private key to use that SSH key + for the session authentication. + Set ``ssh_key`` to `True` to force passing the current SSH authentication socket + to the desired ``hostname``. + + Set ``ssh_config`` to a file path string of an SSH client config file to pass that + file to the client to handle itself. You may set any options you wish in here, however + doing so will require you to post extra information that you may not want to if you + run into issues. + + Alter the ``cmd`` to change the ssh client used, or to prepend it with network + namespaces. For example ```cmd="ip netns exec vlan2 ssh"``` to execute the ssh in + network namespace named ```vlan```. + ''' + + session_regex_array = ["(?i)are you sure you want to continue connecting", original_prompt, password_regex, "(?i)permission denied", "(?i)terminal type", TIMEOUT] + session_init_regex_array = [] + session_init_regex_array.extend(session_regex_array) + session_init_regex_array.extend(["(?i)connection closed by remote host", EOF]) + + ssh_options = ''.join([" -o '%s=%s'" % (o, v) for (o, v) in self.options.items()]) + if quiet: + ssh_options = ssh_options + ' -q' + if not check_local_ip: + ssh_options = ssh_options + " -o'NoHostAuthenticationForLocalhost=yes'" + if self.force_password: + ssh_options = ssh_options + ' ' + self.SSH_OPTS + if ssh_config is not None: + if spawn_local_ssh and not os.path.isfile(ssh_config): + raise ExceptionPxssh('SSH config does not exist or is not a file.') + ssh_options = ssh_options + ' -F ' + ssh_config + if port is not None: + ssh_options = ssh_options + ' -p %s'%(str(port)) + if ssh_key is not None: + # Allow forwarding our SSH key to the current session + if ssh_key==True: + ssh_options = ssh_options + ' -A' + else: + if spawn_local_ssh and not os.path.isfile(ssh_key): + raise ExceptionPxssh('private ssh key does not exist or is not a file.') + ssh_options = ssh_options + ' -i %s' % (ssh_key) + + # SSH tunnels, make sure you know what you're putting into the lists + # under each heading. Do not expect these to open 100% of the time, + # The port you're requesting might be bound. + # + # The structure should be like this: + # { 'local': ['2424:localhost:22'], # Local SSH tunnels + # 'remote': ['2525:localhost:22'], # Remote SSH tunnels + # 'dynamic': [8888] } # Dynamic/SOCKS tunnels + if ssh_tunnels!={} and isinstance({},type(ssh_tunnels)): + tunnel_types = { + 'local':'L', + 'remote':'R', + 'dynamic':'D' + } + for tunnel_type in tunnel_types: + cmd_type = tunnel_types[tunnel_type] + if tunnel_type in ssh_tunnels: + tunnels = ssh_tunnels[tunnel_type] + for tunnel in tunnels: + if spawn_local_ssh==False: + tunnel = quote(str(tunnel)) + ssh_options = ssh_options + ' -' + cmd_type + ' ' + str(tunnel) + + if username is not None: + ssh_options = ssh_options + ' -l ' + username + elif ssh_config is None: + raise TypeError('login() needs either a username or an ssh_config') + else: # make sure ssh_config has an entry for the server with a username + with open(ssh_config, 'rt') as f: + lines = [l.strip() for l in f.readlines()] + + server_regex = r'^Host\s+%s\s*$' % server + user_regex = r'^User\s+\w+\s*$' + config_has_server = False + server_has_username = False + for line in lines: + if not config_has_server and re.match(server_regex, line, re.IGNORECASE): + config_has_server = True + elif config_has_server and 'hostname' in line.lower(): + pass + elif config_has_server and 'host' in line.lower(): + server_has_username = False # insurance + break # we have left the relevant section + elif config_has_server and re.match(user_regex, line, re.IGNORECASE): + server_has_username = True + break + + if lines: + del line + + del lines + + if not config_has_server: + raise TypeError('login() ssh_config has no Host entry for %s' % server) + elif not server_has_username: + raise TypeError('login() ssh_config has no user entry for %s' % server) + + cmd += " %s %s" % (ssh_options, server) + if self.debug_command_string: + return(cmd) + + # Are we asking for a local ssh command or to spawn one in another session? + if spawn_local_ssh: + spawn._spawn(self, cmd) + else: + self.sendline(cmd) + + # This does not distinguish between a remote server 'password' prompt + # and a local ssh 'passphrase' prompt (for unlocking a private key). + i = self.expect(session_init_regex_array, timeout=login_timeout) + + # First phase + if i==0: + # New certificate -- always accept it. + # This is what you get if SSH does not have the remote host's + # public key stored in the 'known_hosts' cache. + self.sendline("yes") + i = self.expect(session_regex_array) + if i==2: # password or passphrase + self.sendline(password) + i = self.expect(session_regex_array) + if i==4: + self.sendline(terminal_type) + i = self.expect(session_regex_array) + if i==7: + self.close() + raise ExceptionPxssh('Could not establish connection to host') + + # Second phase + if i==0: + # This is weird. This should not happen twice in a row. + self.close() + raise ExceptionPxssh('Weird error. Got "are you sure" prompt twice.') + elif i==1: # can occur if you have a public key pair set to authenticate. + ### TODO: May NOT be OK if expect() got tricked and matched a false prompt. + pass + elif i==2: # password prompt again + # For incorrect passwords, some ssh servers will + # ask for the password again, others return 'denied' right away. + # If we get the password prompt again then this means + # we didn't get the password right the first time. + self.close() + raise ExceptionPxssh('password refused') + elif i==3: # permission denied -- password was bad. + self.close() + raise ExceptionPxssh('permission denied') + elif i==4: # terminal type again? WTF? + self.close() + raise ExceptionPxssh('Weird error. Got "terminal type" prompt twice.') + elif i==5: # Timeout + #This is tricky... I presume that we are at the command-line prompt. + #It may be that the shell prompt was so weird that we couldn't match + #it. Or it may be that we couldn't log in for some other reason. I + #can't be sure, but it's safe to guess that we did login because if + #I presume wrong and we are not logged in then this should be caught + #later when I try to set the shell prompt. + pass + elif i==6: # Connection closed by remote host + self.close() + raise ExceptionPxssh('connection closed') + else: # Unexpected + self.close() + raise ExceptionPxssh('unexpected login response') + if sync_original_prompt: + if not self.sync_original_prompt(sync_multiplier): + self.close() + raise ExceptionPxssh('could not synchronize with original prompt') + # We appear to be in. + # set shell prompt to something unique. + if auto_prompt_reset: + if not self.set_unique_prompt(): + self.close() + raise ExceptionPxssh('could not set shell prompt ' + '(received: %r, expected: %r).' % ( + self.before, self.PROMPT,)) + return True + + def logout (self): + '''Sends exit to the remote shell. + + If there are stopped jobs then this automatically sends exit twice. + ''' + self.sendline("exit") + index = self.expect([EOF, "(?i)there are stopped jobs"]) + if index==1: + self.sendline("exit") + self.expect(EOF) + self.close() + + def prompt(self, timeout=-1): + '''Match the next shell prompt. + + This is little more than a short-cut to the :meth:`~pexpect.spawn.expect` + method. Note that if you called :meth:`login` with + ``auto_prompt_reset=False``, then before calling :meth:`prompt` you must + set the :attr:`PROMPT` attribute to a regex that it will use for + matching the prompt. + + Calling :meth:`prompt` will erase the contents of the :attr:`before` + attribute even if no prompt is ever matched. If timeout is not given or + it is set to -1 then self.timeout is used. + + :return: True if the shell prompt was matched, False if the timeout was + reached. + ''' + + if timeout == -1: + timeout = self.timeout + i = self.expect([self.PROMPT, TIMEOUT], timeout=timeout) + if i==1: + return False + return True + + def set_unique_prompt(self): + '''This sets the remote prompt to something more unique than ``#`` or ``$``. + This makes it easier for the :meth:`prompt` method to match the shell prompt + unambiguously. This method is called automatically by the :meth:`login` + method, but you may want to call it manually if you somehow reset the + shell prompt. For example, if you 'su' to a different user then you + will need to manually reset the prompt. This sends shell commands to + the remote host to set the prompt, so this assumes the remote host is + ready to receive commands. + + Alternatively, you may use your own prompt pattern. In this case you + should call :meth:`login` with ``auto_prompt_reset=False``; then set the + :attr:`PROMPT` attribute to a regular expression. After that, the + :meth:`prompt` method will try to match your prompt pattern. + ''' + + self.sendline("unset PROMPT_COMMAND") + self.sendline(self.PROMPT_SET_SH) # sh-style + i = self.expect ([TIMEOUT, self.PROMPT], timeout=10) + if i == 0: # csh-style + self.sendline(self.PROMPT_SET_CSH) + i = self.expect([TIMEOUT, self.PROMPT], timeout=10) + if i == 0: # zsh-style + self.sendline(self.PROMPT_SET_ZSH) + i = self.expect([TIMEOUT, self.PROMPT], timeout=10) + if i == 0: + return False + return True + +# vi:ts=4:sw=4:expandtab:ft=python: diff --git a/lib/python3.12/site-packages/pexpect/replwrap.py b/lib/python3.12/site-packages/pexpect/replwrap.py new file mode 100644 index 0000000000000000000000000000000000000000..08dbd5e8692fa812821a1204c0a5675c5fd4c8df --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/replwrap.py @@ -0,0 +1,136 @@ +"""Generic wrapper for read-eval-print-loops, a.k.a. interactive shells +""" +import os.path +import signal +import sys + +import pexpect + +PY3 = (sys.version_info[0] >= 3) + +if PY3: + basestring = str + +PEXPECT_PROMPT = u'[PEXPECT_PROMPT>' +PEXPECT_CONTINUATION_PROMPT = u'[PEXPECT_PROMPT+' + +class REPLWrapper(object): + """Wrapper for a REPL. + + :param cmd_or_spawn: This can either be an instance of :class:`pexpect.spawn` + in which a REPL has already been started, or a str command to start a new + REPL process. + :param str orig_prompt: The prompt to expect at first. + :param str prompt_change: A command to change the prompt to something more + unique. If this is ``None``, the prompt will not be changed. This will + be formatted with the new and continuation prompts as positional + parameters, so you can use ``{}`` style formatting to insert them into + the command. + :param str new_prompt: The more unique prompt to expect after the change. + :param str extra_init_cmd: Commands to do extra initialisation, such as + disabling pagers. + """ + def __init__(self, cmd_or_spawn, orig_prompt, prompt_change, + new_prompt=PEXPECT_PROMPT, + continuation_prompt=PEXPECT_CONTINUATION_PROMPT, + extra_init_cmd=None): + if isinstance(cmd_or_spawn, basestring): + self.child = pexpect.spawn(cmd_or_spawn, echo=False, encoding='utf-8') + else: + self.child = cmd_or_spawn + if self.child.echo: + # Existing spawn instance has echo enabled, disable it + # to prevent our input from being repeated to output. + self.child.setecho(False) + self.child.waitnoecho() + + if prompt_change is None: + self.prompt = orig_prompt + else: + self.set_prompt(orig_prompt, + prompt_change.format(new_prompt, continuation_prompt)) + self.prompt = new_prompt + self.continuation_prompt = continuation_prompt + + self._expect_prompt() + + if extra_init_cmd is not None: + self.run_command(extra_init_cmd) + + def set_prompt(self, orig_prompt, prompt_change): + self.child.expect(orig_prompt) + self.child.sendline(prompt_change) + + def _expect_prompt(self, timeout=-1, async_=False): + return self.child.expect_exact([self.prompt, self.continuation_prompt], + timeout=timeout, async_=async_) + + def run_command(self, command, timeout=-1, async_=False): + """Send a command to the REPL, wait for and return output. + + :param str command: The command to send. Trailing newlines are not needed. + This should be a complete block of input that will trigger execution; + if a continuation prompt is found after sending input, :exc:`ValueError` + will be raised. + :param int timeout: How long to wait for the next prompt. -1 means the + default from the :class:`pexpect.spawn` object (default 30 seconds). + None means to wait indefinitely. + :param bool async_: On Python 3.4, or Python 3.3 with asyncio + installed, passing ``async_=True`` will make this return an + :mod:`asyncio` Future, which you can yield from to get the same + result that this method would normally give directly. + """ + # Split up multiline commands and feed them in bit-by-bit + cmdlines = command.splitlines() + # splitlines ignores trailing newlines - add it back in manually + if command.endswith('\n'): + cmdlines.append('') + if not cmdlines: + raise ValueError("No command was given") + + if async_: + from ._async import repl_run_command_async + return repl_run_command_async(self, cmdlines, timeout) + + res = [] + self.child.sendline(cmdlines[0]) + for line in cmdlines[1:]: + self._expect_prompt(timeout=timeout) + res.append(self.child.before) + self.child.sendline(line) + + # Command was fully submitted, now wait for the next prompt + if self._expect_prompt(timeout=timeout) == 1: + # We got the continuation prompt - command was incomplete + self.child.kill(signal.SIGINT) + self._expect_prompt(timeout=1) + raise ValueError("Continuation prompt found - input was incomplete:\n" + + command) + return u''.join(res + [self.child.before]) + +def python(command=sys.executable): + """Start a Python shell and return a :class:`REPLWrapper` object.""" + return REPLWrapper(command, u">>> ", u"import sys; sys.ps1={0!r}; sys.ps2={1!r}") + +def _repl_sh(command, args, non_printable_insert): + child = pexpect.spawn(command, args, echo=False, encoding='utf-8') + + # If the user runs 'env', the value of PS1 will be in the output. To avoid + # replwrap seeing that as the next prompt, we'll embed the marker characters + # for invisible characters in the prompt; these show up when inspecting the + # environment variable, but not when bash displays the prompt. + ps1 = PEXPECT_PROMPT[:5] + non_printable_insert + PEXPECT_PROMPT[5:] + ps2 = PEXPECT_CONTINUATION_PROMPT[:5] + non_printable_insert + PEXPECT_CONTINUATION_PROMPT[5:] + prompt_change = u"PS1='{0}' PS2='{1}' PROMPT_COMMAND=''".format(ps1, ps2) + + return REPLWrapper(child, u'\\$', prompt_change, + extra_init_cmd="export PAGER=cat") + +def bash(command="bash"): + """Start a bash shell and return a :class:`REPLWrapper` object.""" + bashrc = os.path.join(os.path.dirname(__file__), 'bashrc.sh') + return _repl_sh(command, ['--rcfile', bashrc], non_printable_insert='\\[\\]') + +def zsh(command="zsh", args=("--no-rcs", "-V", "+Z")): + """Start a zsh shell and return a :class:`REPLWrapper` object.""" + return _repl_sh(command, list(args), non_printable_insert='%(!..)') diff --git a/lib/python3.12/site-packages/pexpect/run.py b/lib/python3.12/site-packages/pexpect/run.py new file mode 100644 index 0000000000000000000000000000000000000000..5695ab7f7b4811dffdddadf6e2310a9879f8817c --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/run.py @@ -0,0 +1,157 @@ +import sys +import types + +from .exceptions import EOF, TIMEOUT +from .pty_spawn import spawn + +def run(command, timeout=30, withexitstatus=False, events=None, + extra_args=None, logfile=None, cwd=None, env=None, **kwargs): + + ''' + This function runs the given command; waits for it to finish; then + returns all output as a string. STDERR is included in output. If the full + path to the command is not given then the path is searched. + + Note that lines are terminated by CR/LF (\\r\\n) combination even on + UNIX-like systems because this is the standard for pseudottys. If you set + 'withexitstatus' to true, then run will return a tuple of (command_output, + exitstatus). If 'withexitstatus' is false then this returns just + command_output. + + The run() function can often be used instead of creating a spawn instance. + For example, the following code uses spawn:: + + from pexpect import * + child = spawn('scp foo user@example.com:.') + child.expect('(?i)password') + child.sendline(mypassword) + + The previous code can be replace with the following:: + + from pexpect import * + run('scp foo user@example.com:.', events={'(?i)password': mypassword}) + + **Examples** + + Start the apache daemon on the local machine:: + + from pexpect import * + run("/usr/local/apache/bin/apachectl start") + + Check in a file using SVN:: + + from pexpect import * + run("svn ci -m 'automatic commit' my_file.py") + + Run a command and capture exit status:: + + from pexpect import * + (command_output, exitstatus) = run('ls -l /bin', withexitstatus=1) + + The following will run SSH and execute 'ls -l' on the remote machine. The + password 'secret' will be sent if the '(?i)password' pattern is ever seen:: + + run("ssh username@machine.example.com 'ls -l'", + events={'(?i)password':'secret\\n'}) + + This will start mencoder to rip a video from DVD. This will also display + progress ticks every 5 seconds as it runs. For example:: + + from pexpect import * + def print_ticks(d): + print d['event_count'], + run("mencoder dvd://1 -o video.avi -oac copy -ovc copy", + events={TIMEOUT:print_ticks}, timeout=5) + + The 'events' argument should be either a dictionary or a tuple list that + contains patterns and responses. Whenever one of the patterns is seen + in the command output, run() will send the associated response string. + So, run() in the above example can be also written as:: + + run("mencoder dvd://1 -o video.avi -oac copy -ovc copy", + events=[(TIMEOUT,print_ticks)], timeout=5) + + Use a tuple list for events if the command output requires a delicate + control over what pattern should be matched, since the tuple list is passed + to pexpect() as its pattern list, with the order of patterns preserved. + + Note that you should put newlines in your string if Enter is necessary. + + Like the example above, the responses may also contain a callback, either + a function or method. It should accept a dictionary value as an argument. + The dictionary contains all the locals from the run() function, so you can + access the child spawn object or any other variable defined in run() + (event_count, child, and extra_args are the most useful). A callback may + return True to stop the current run process. Otherwise run() continues + until the next event. A callback may also return a string which will be + sent to the child. 'extra_args' is not used by directly run(). It provides + a way to pass data to a callback function through run() through the locals + dictionary passed to a callback. + + Like :class:`spawn`, passing *encoding* will make it work with unicode + instead of bytes. You can pass *codec_errors* to control how errors in + encoding and decoding are handled. + ''' + if timeout == -1: + child = spawn(command, maxread=2000, logfile=logfile, cwd=cwd, env=env, + **kwargs) + else: + child = spawn(command, timeout=timeout, maxread=2000, logfile=logfile, + cwd=cwd, env=env, **kwargs) + if isinstance(events, list): + patterns= [x for x,y in events] + responses = [y for x,y in events] + elif isinstance(events, dict): + patterns = list(events.keys()) + responses = list(events.values()) + else: + # This assumes EOF or TIMEOUT will eventually cause run to terminate. + patterns = None + responses = None + child_result_list = [] + event_count = 0 + while True: + try: + index = child.expect(patterns) + if isinstance(child.after, child.allowed_string_types): + child_result_list.append(child.before + child.after) + else: + # child.after may have been a TIMEOUT or EOF, + # which we don't want appended to the list. + child_result_list.append(child.before) + if isinstance(responses[index], child.allowed_string_types): + child.send(responses[index]) + elif (isinstance(responses[index], types.FunctionType) or + isinstance(responses[index], types.MethodType)): + callback_result = responses[index](locals()) + sys.stdout.flush() + if isinstance(callback_result, child.allowed_string_types): + child.send(callback_result) + elif callback_result: + break + else: + raise TypeError("parameter `event' at index {index} must be " + "a string, method, or function: {value!r}" + .format(index=index, value=responses[index])) + event_count = event_count + 1 + except TIMEOUT: + child_result_list.append(child.before) + break + except EOF: + child_result_list.append(child.before) + break + child_result = child.string_type().join(child_result_list) + if withexitstatus: + child.close() + return (child_result, child.exitstatus) + else: + return child_result + +def runu(command, timeout=30, withexitstatus=False, events=None, + extra_args=None, logfile=None, cwd=None, env=None, **kwargs): + """Deprecated: pass encoding to run() instead. + """ + kwargs.setdefault('encoding', 'utf-8') + return run(command, timeout=timeout, withexitstatus=withexitstatus, + events=events, extra_args=extra_args, logfile=logfile, cwd=cwd, + env=env, **kwargs) diff --git a/lib/python3.12/site-packages/pexpect/screen.py b/lib/python3.12/site-packages/pexpect/screen.py new file mode 100644 index 0000000000000000000000000000000000000000..79f95c4e54273c3ca4e00623fd55cce516040392 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/screen.py @@ -0,0 +1,431 @@ +'''This implements a virtual screen. This is used to support ANSI terminal +emulation. The screen representation and state is implemented in this class. +Most of the methods are inspired by ANSI screen control codes. The +:class:`~pexpect.ANSI.ANSI` class extends this class to add parsing of ANSI +escape codes. + +PEXPECT LICENSE + + This license is approved by the OSI and FSF as GPL-compatible. + http://opensource.org/licenses/isc-license.txt + + Copyright (c) 2012, Noah Spurrier + PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY + PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE + COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +''' + +import codecs +import copy +import sys + +import warnings + +warnings.warn(("pexpect.screen and pexpect.ANSI are deprecated. " + "We recommend using pyte to emulate a terminal screen: " + "https://pypi.python.org/pypi/pyte"), + stacklevel=2) + +NUL = 0 # Fill character; ignored on input. +ENQ = 5 # Transmit answerback message. +BEL = 7 # Ring the bell. +BS = 8 # Move cursor left. +HT = 9 # Move cursor to next tab stop. +LF = 10 # Line feed. +VT = 11 # Same as LF. +FF = 12 # Same as LF. +CR = 13 # Move cursor to left margin or newline. +SO = 14 # Invoke G1 character set. +SI = 15 # Invoke G0 character set. +XON = 17 # Resume transmission. +XOFF = 19 # Halt transmission. +CAN = 24 # Cancel escape sequence. +SUB = 26 # Same as CAN. +ESC = 27 # Introduce a control sequence. +DEL = 127 # Fill character; ignored on input. +SPACE = u' ' # Space or blank character. + +PY3 = (sys.version_info[0] >= 3) +if PY3: + unicode = str + +def constrain (n, min, max): + + '''This returns a number, n constrained to the min and max bounds. ''' + + if n < min: + return min + if n > max: + return max + return n + +class screen: + '''This object maintains the state of a virtual text screen as a + rectangular array. This maintains a virtual cursor position and handles + scrolling as characters are added. This supports most of the methods needed + by an ANSI text screen. Row and column indexes are 1-based (not zero-based, + like arrays). + + Characters are represented internally using unicode. Methods that accept + input characters, when passed 'bytes' (which in Python 2 is equivalent to + 'str'), convert them from the encoding specified in the 'encoding' + parameter to the constructor. Methods that return screen contents return + unicode strings, with the exception of __str__() under Python 2. Passing + ``encoding=None`` limits the API to only accept unicode input, so passing + bytes in will raise :exc:`TypeError`. + ''' + def __init__(self, r=24, c=80, encoding='latin-1', encoding_errors='replace'): + '''This initializes a blank screen of the given dimensions.''' + + self.rows = r + self.cols = c + self.encoding = encoding + self.encoding_errors = encoding_errors + if encoding is not None: + self.decoder = codecs.getincrementaldecoder(encoding)(encoding_errors) + else: + self.decoder = None + self.cur_r = 1 + self.cur_c = 1 + self.cur_saved_r = 1 + self.cur_saved_c = 1 + self.scroll_row_start = 1 + self.scroll_row_end = self.rows + self.w = [ [SPACE] * self.cols for _ in range(self.rows)] + + def _decode(self, s): + '''This converts from the external coding system (as passed to + the constructor) to the internal one (unicode). ''' + if self.decoder is not None: + return self.decoder.decode(s) + else: + raise TypeError("This screen was constructed with encoding=None, " + "so it does not handle bytes.") + + def _unicode(self): + '''This returns a printable representation of the screen as a unicode + string (which, under Python 3.x, is the same as 'str'). The end of each + screen line is terminated by a newline.''' + + return u'\n'.join ([ u''.join(c) for c in self.w ]) + + if PY3: + __str__ = _unicode + else: + __unicode__ = _unicode + + def __str__(self): + '''This returns a printable representation of the screen. The end of + each screen line is terminated by a newline. ''' + encoding = self.encoding or 'ascii' + return self._unicode().encode(encoding, 'replace') + + def dump (self): + '''This returns a copy of the screen as a unicode string. This is similar to + __str__/__unicode__ except that lines are not terminated with line + feeds.''' + + return u''.join ([ u''.join(c) for c in self.w ]) + + def pretty (self): + '''This returns a copy of the screen as a unicode string with an ASCII + text box around the screen border. This is similar to + __str__/__unicode__ except that it adds a box.''' + + top_bot = u'+' + u'-'*self.cols + u'+\n' + return top_bot + u'\n'.join([u'|'+line+u'|' for line in unicode(self).split(u'\n')]) + u'\n' + top_bot + + def fill (self, ch=SPACE): + + if isinstance(ch, bytes): + ch = self._decode(ch) + + self.fill_region (1,1,self.rows,self.cols, ch) + + def fill_region (self, rs,cs, re,ce, ch=SPACE): + + if isinstance(ch, bytes): + ch = self._decode(ch) + + rs = constrain (rs, 1, self.rows) + re = constrain (re, 1, self.rows) + cs = constrain (cs, 1, self.cols) + ce = constrain (ce, 1, self.cols) + if rs > re: + rs, re = re, rs + if cs > ce: + cs, ce = ce, cs + for r in range (rs, re+1): + for c in range (cs, ce + 1): + self.put_abs (r,c,ch) + + def cr (self): + '''This moves the cursor to the beginning (col 1) of the current row. + ''' + + self.cursor_home (self.cur_r, 1) + + def lf (self): + '''This moves the cursor down with scrolling. + ''' + + old_r = self.cur_r + self.cursor_down() + if old_r == self.cur_r: + self.scroll_up () + self.erase_line() + + def crlf (self): + '''This advances the cursor with CRLF properties. + The cursor will line wrap and the screen may scroll. + ''' + + self.cr () + self.lf () + + def newline (self): + '''This is an alias for crlf(). + ''' + + self.crlf() + + def put_abs (self, r, c, ch): + '''Screen array starts at 1 index.''' + + r = constrain (r, 1, self.rows) + c = constrain (c, 1, self.cols) + if isinstance(ch, bytes): + ch = self._decode(ch)[0] + else: + ch = ch[0] + self.w[r-1][c-1] = ch + + def put (self, ch): + '''This puts a characters at the current cursor position. + ''' + + if isinstance(ch, bytes): + ch = self._decode(ch) + + self.put_abs (self.cur_r, self.cur_c, ch) + + def insert_abs (self, r, c, ch): + '''This inserts a character at (r,c). Everything under + and to the right is shifted right one character. + The last character of the line is lost. + ''' + + if isinstance(ch, bytes): + ch = self._decode(ch) + + r = constrain (r, 1, self.rows) + c = constrain (c, 1, self.cols) + for ci in range (self.cols, c, -1): + self.put_abs (r,ci, self.get_abs(r,ci-1)) + self.put_abs (r,c,ch) + + def insert (self, ch): + + if isinstance(ch, bytes): + ch = self._decode(ch) + + self.insert_abs (self.cur_r, self.cur_c, ch) + + def get_abs (self, r, c): + + r = constrain (r, 1, self.rows) + c = constrain (c, 1, self.cols) + return self.w[r-1][c-1] + + def get (self): + + self.get_abs (self.cur_r, self.cur_c) + + def get_region (self, rs,cs, re,ce): + '''This returns a list of lines representing the region. + ''' + + rs = constrain (rs, 1, self.rows) + re = constrain (re, 1, self.rows) + cs = constrain (cs, 1, self.cols) + ce = constrain (ce, 1, self.cols) + if rs > re: + rs, re = re, rs + if cs > ce: + cs, ce = ce, cs + sc = [] + for r in range (rs, re+1): + line = u'' + for c in range (cs, ce + 1): + ch = self.get_abs (r,c) + line = line + ch + sc.append (line) + return sc + + def cursor_constrain (self): + '''This keeps the cursor within the screen area. + ''' + + self.cur_r = constrain (self.cur_r, 1, self.rows) + self.cur_c = constrain (self.cur_c, 1, self.cols) + + def cursor_home (self, r=1, c=1): # [{ROW};{COLUMN}H + + self.cur_r = r + self.cur_c = c + self.cursor_constrain () + + def cursor_back (self,count=1): # [{COUNT}D (not confused with down) + + self.cur_c = self.cur_c - count + self.cursor_constrain () + + def cursor_down (self,count=1): # [{COUNT}B (not confused with back) + + self.cur_r = self.cur_r + count + self.cursor_constrain () + + def cursor_forward (self,count=1): # [{COUNT}C + + self.cur_c = self.cur_c + count + self.cursor_constrain () + + def cursor_up (self,count=1): # [{COUNT}A + + self.cur_r = self.cur_r - count + self.cursor_constrain () + + def cursor_up_reverse (self): # M (called RI -- Reverse Index) + + old_r = self.cur_r + self.cursor_up() + if old_r == self.cur_r: + self.scroll_up() + + def cursor_force_position (self, r, c): # [{ROW};{COLUMN}f + '''Identical to Cursor Home.''' + + self.cursor_home (r, c) + + def cursor_save (self): # [s + '''Save current cursor position.''' + + self.cursor_save_attrs() + + def cursor_unsave (self): # [u + '''Restores cursor position after a Save Cursor.''' + + self.cursor_restore_attrs() + + def cursor_save_attrs (self): # 7 + '''Save current cursor position.''' + + self.cur_saved_r = self.cur_r + self.cur_saved_c = self.cur_c + + def cursor_restore_attrs (self): # 8 + '''Restores cursor position after a Save Cursor.''' + + self.cursor_home (self.cur_saved_r, self.cur_saved_c) + + def scroll_constrain (self): + '''This keeps the scroll region within the screen region.''' + + if self.scroll_row_start <= 0: + self.scroll_row_start = 1 + if self.scroll_row_end > self.rows: + self.scroll_row_end = self.rows + + def scroll_screen (self): # [r + '''Enable scrolling for entire display.''' + + self.scroll_row_start = 1 + self.scroll_row_end = self.rows + + def scroll_screen_rows (self, rs, re): # [{start};{end}r + '''Enable scrolling from row {start} to row {end}.''' + + self.scroll_row_start = rs + self.scroll_row_end = re + self.scroll_constrain() + + def scroll_down (self): # D + '''Scroll display down one line.''' + + # Screen is indexed from 1, but arrays are indexed from 0. + s = self.scroll_row_start - 1 + e = self.scroll_row_end - 1 + self.w[s+1:e+1] = copy.deepcopy(self.w[s:e]) + + def scroll_up (self): # M + '''Scroll display up one line.''' + + # Screen is indexed from 1, but arrays are indexed from 0. + s = self.scroll_row_start - 1 + e = self.scroll_row_end - 1 + self.w[s:e] = copy.deepcopy(self.w[s+1:e+1]) + + def erase_end_of_line (self): # [0K -or- [K + '''Erases from the current cursor position to the end of the current + line.''' + + self.fill_region (self.cur_r, self.cur_c, self.cur_r, self.cols) + + def erase_start_of_line (self): # [1K + '''Erases from the current cursor position to the start of the current + line.''' + + self.fill_region (self.cur_r, 1, self.cur_r, self.cur_c) + + def erase_line (self): # [2K + '''Erases the entire current line.''' + + self.fill_region (self.cur_r, 1, self.cur_r, self.cols) + + def erase_down (self): # [0J -or- [J + '''Erases the screen from the current line down to the bottom of the + screen.''' + + self.erase_end_of_line () + self.fill_region (self.cur_r + 1, 1, self.rows, self.cols) + + def erase_up (self): # [1J + '''Erases the screen from the current line up to the top of the + screen.''' + + self.erase_start_of_line () + self.fill_region (self.cur_r-1, 1, 1, self.cols) + + def erase_screen (self): # [2J + '''Erases the screen with the background color.''' + + self.fill () + + def set_tab (self): # H + '''Sets a tab at the current position.''' + + pass + + def clear_tab (self): # [g + '''Clears tab at the current position.''' + + pass + + def clear_all_tabs (self): # [3g + '''Clears all tabs.''' + + pass + +# Insert line Esc [ Pn L +# Delete line Esc [ Pn M +# Delete character Esc [ Pn P +# Scrolling region Esc [ Pn(top);Pn(bot) r + diff --git a/lib/python3.12/site-packages/pexpect/socket_pexpect.py b/lib/python3.12/site-packages/pexpect/socket_pexpect.py new file mode 100644 index 0000000000000000000000000000000000000000..cb11ac225891f090f9c4fb312d0707210d0a1b55 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/socket_pexpect.py @@ -0,0 +1,145 @@ +"""This is like :mod:`pexpect`, but it will work with any socket that you +pass it. You are responsible for opening and closing the socket. + +PEXPECT LICENSE + + This license is approved by the OSI and FSF as GPL-compatible. + http://opensource.org/licenses/isc-license.txt + + Copyright (c) 2012, Noah Spurrier + PERMISSION TO USE, COPY, MODIFY, AND/OR DISTRIBUTE THIS SOFTWARE FOR ANY + PURPOSE WITH OR WITHOUT FEE IS HEREBY GRANTED, PROVIDED THAT THE ABOVE + COPYRIGHT NOTICE AND THIS PERMISSION NOTICE APPEAR IN ALL COPIES. + THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +""" + +import socket +from contextlib import contextmanager + +from .exceptions import TIMEOUT, EOF +from .spawnbase import SpawnBase + +__all__ = ["SocketSpawn"] + + +class SocketSpawn(SpawnBase): + """This is like :mod:`pexpect.fdpexpect` but uses the cross-platform python socket api, + rather than the unix-specific file descriptor api. Thus, it works with + remote connections on both unix and windows.""" + + def __init__( + self, + socket: socket.socket, + args=None, + timeout=30, + maxread=2000, + searchwindowsize=None, + logfile=None, + encoding=None, + codec_errors="strict", + use_poll=False, + ): + """This takes an open socket.""" + + self.args = None + self.command = None + SpawnBase.__init__( + self, + timeout, + maxread, + searchwindowsize, + logfile, + encoding=encoding, + codec_errors=codec_errors, + ) + self.socket = socket + self.child_fd = socket.fileno() + self.closed = False + self.name = "" % socket + self.use_poll = use_poll + + def close(self): + """Close the socket. + + Calling this method a second time does nothing, but if the file + descriptor was closed elsewhere, :class:`OSError` will be raised. + """ + if self.child_fd == -1: + return + + self.flush() + self.socket.shutdown(socket.SHUT_RDWR) + self.socket.close() + self.child_fd = -1 + self.closed = True + + def isalive(self): + """ Alive if the fileno is valid """ + return self.socket.fileno() >= 0 + + def send(self, s) -> int: + """Write to socket, return number of bytes written""" + s = self._coerce_send_string(s) + self._log(s, "send") + + b = self._encoder.encode(s, final=False) + self.socket.sendall(b) + return len(b) + + def sendline(self, s) -> int: + """Write to socket with trailing newline, return number of bytes written""" + s = self._coerce_send_string(s) + return self.send(s + self.linesep) + + def write(self, s): + """Write to socket, return None""" + self.send(s) + + def writelines(self, sequence): + "Call self.write() for each item in sequence" + for s in sequence: + self.write(s) + + @contextmanager + def _timeout(self, timeout): + saved_timeout = self.socket.gettimeout() + try: + self.socket.settimeout(timeout) + yield + finally: + self.socket.settimeout(saved_timeout) + + def read_nonblocking(self, size=1, timeout=-1): + """ + Read from the file descriptor and return the result as a string. + + The read_nonblocking method of :class:`SpawnBase` assumes that a call + to os.read will not block (timeout parameter is ignored). This is not + the case for POSIX file-like objects such as sockets and serial ports. + + Use :func:`select.select`, timeout is implemented conditionally for + POSIX systems. + + :param int size: Read at most *size* bytes. + :param int timeout: Wait timeout seconds for file descriptor to be + ready to read. When -1 (default), use self.timeout. When 0, poll. + :return: String containing the bytes read + """ + if timeout == -1: + timeout = self.timeout + try: + with self._timeout(timeout): + s = self.socket.recv(size) + if s == b'': + self.flag_eof = True + raise EOF("Socket closed") + return s + except socket.timeout: + raise TIMEOUT("Timeout exceeded.") diff --git a/lib/python3.12/site-packages/pexpect/spawnbase.py b/lib/python3.12/site-packages/pexpect/spawnbase.py new file mode 100644 index 0000000000000000000000000000000000000000..abf8071ec152e68d4d30265360b185e24c1a6231 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/spawnbase.py @@ -0,0 +1,536 @@ +from io import StringIO, BytesIO +import codecs +import os +import sys +import re +import errno +from .exceptions import ExceptionPexpect, EOF, TIMEOUT +from .expect import Expecter, searcher_string, searcher_re + +PY3 = (sys.version_info[0] >= 3) +text_type = str if PY3 else unicode + +class _NullCoder(object): + """Pass bytes through unchanged.""" + @staticmethod + def encode(b, final=False): + return b + + @staticmethod + def decode(b, final=False): + return b + +class SpawnBase(object): + """A base class providing the backwards-compatible spawn API for Pexpect. + + This should not be instantiated directly: use :class:`pexpect.spawn` or + :class:`pexpect.fdpexpect.fdspawn`. + """ + encoding = None + pid = None + flag_eof = False + + def __init__(self, timeout=30, maxread=2000, searchwindowsize=None, + logfile=None, encoding=None, codec_errors='strict'): + self.stdin = sys.stdin + self.stdout = sys.stdout + self.stderr = sys.stderr + + self.searcher = None + self.ignorecase = False + self.before = None + self.after = None + self.match = None + self.match_index = None + self.terminated = True + self.exitstatus = None + self.signalstatus = None + # status returned by os.waitpid + self.status = None + # the child file descriptor is initially closed + self.child_fd = -1 + self.timeout = timeout + self.delimiter = EOF + self.logfile = logfile + # input from child (read_nonblocking) + self.logfile_read = None + # output to send (send, sendline) + self.logfile_send = None + # max bytes to read at one time into buffer + self.maxread = maxread + # Data before searchwindowsize point is preserved, but not searched. + self.searchwindowsize = searchwindowsize + # Delay used before sending data to child. Time in seconds. + # Set this to None to skip the time.sleep() call completely. + self.delaybeforesend = 0.05 + # Used by close() to give kernel time to update process status. + # Time in seconds. + self.delayafterclose = 0.1 + # Used by terminate() to give kernel time to update process status. + # Time in seconds. + self.delayafterterminate = 0.1 + # Delay in seconds to sleep after each call to read_nonblocking(). + # Set this to None to skip the time.sleep() call completely: that + # would restore the behavior from pexpect-2.0 (for performance + # reasons or because you don't want to release Python's global + # interpreter lock). + self.delayafterread = 0.0001 + self.softspace = False + self.name = '<' + repr(self) + '>' + self.closed = True + + # Unicode interface + self.encoding = encoding + self.codec_errors = codec_errors + if encoding is None: + # bytes mode (accepts some unicode for backwards compatibility) + self._encoder = self._decoder = _NullCoder() + self.string_type = bytes + self.buffer_type = BytesIO + self.crlf = b'\r\n' + if PY3: + self.allowed_string_types = (bytes, str) + self.linesep = os.linesep.encode('ascii') + def write_to_stdout(b): + try: + return sys.stdout.buffer.write(b) + except AttributeError: + # If stdout has been replaced, it may not have .buffer + return sys.stdout.write(b.decode('ascii', 'replace')) + self.write_to_stdout = write_to_stdout + else: + self.allowed_string_types = (basestring,) # analysis:ignore + self.linesep = os.linesep + self.write_to_stdout = sys.stdout.write + else: + # unicode mode + self._encoder = codecs.getincrementalencoder(encoding)(codec_errors) + self._decoder = codecs.getincrementaldecoder(encoding)(codec_errors) + self.string_type = text_type + self.buffer_type = StringIO + self.crlf = u'\r\n' + self.allowed_string_types = (text_type, ) + if PY3: + self.linesep = os.linesep + else: + self.linesep = os.linesep.decode('ascii') + # This can handle unicode in both Python 2 and 3 + self.write_to_stdout = sys.stdout.write + # storage for async transport + self.async_pw_transport = None + # This is the read buffer. See maxread. + self._buffer = self.buffer_type() + # The buffer may be trimmed for efficiency reasons. This is the + # untrimmed buffer, used to create the before attribute. + self._before = self.buffer_type() + + def _log(self, s, direction): + if self.logfile is not None: + self.logfile.write(s) + self.logfile.flush() + second_log = self.logfile_send if (direction=='send') else self.logfile_read + if second_log is not None: + second_log.write(s) + second_log.flush() + + # For backwards compatibility, in bytes mode (when encoding is None) + # unicode is accepted for send and expect. Unicode mode is strictly unicode + # only. + def _coerce_expect_string(self, s): + if self.encoding is None and not isinstance(s, bytes): + return s.encode('ascii') + return s + + # In bytes mode, regex patterns should also be of bytes type + def _coerce_expect_re(self, r): + p = r.pattern + if self.encoding is None and not isinstance(p, bytes): + return re.compile(p.encode('utf-8')) + # And vice-versa + elif self.encoding is not None and isinstance(p, bytes): + return re.compile(p.decode('utf-8')) + return r + + def _coerce_send_string(self, s): + if self.encoding is None and not isinstance(s, bytes): + return s.encode('utf-8') + return s + + def _get_buffer(self): + return self._buffer.getvalue() + + def _set_buffer(self, value): + self._buffer = self.buffer_type() + self._buffer.write(value) + + # This property is provided for backwards compatibility (self.buffer used + # to be a string/bytes object) + buffer = property(_get_buffer, _set_buffer) + + def read_nonblocking(self, size=1, timeout=None): + """This reads data from the file descriptor. + + This is a simple implementation suitable for a regular file. Subclasses using ptys or pipes should override it. + + The timeout parameter is ignored. + """ + + try: + s = os.read(self.child_fd, size) + except OSError as err: + if err.args[0] == errno.EIO: + # Linux-style EOF + self.flag_eof = True + raise EOF('End Of File (EOF). Exception style platform.') + raise + if s == b'': + # BSD-style EOF + self.flag_eof = True + raise EOF('End Of File (EOF). Empty string style platform.') + + s = self._decoder.decode(s, final=False) + self._log(s, 'read') + return s + + def _pattern_type_err(self, pattern): + raise TypeError('got {badtype} ({badobj!r}) as pattern, must be one' + ' of: {goodtypes}, pexpect.EOF, pexpect.TIMEOUT'\ + .format(badtype=type(pattern), + badobj=pattern, + goodtypes=', '.join([str(ast)\ + for ast in self.allowed_string_types]) + ) + ) + + def compile_pattern_list(self, patterns): + '''This compiles a pattern-string or a list of pattern-strings. + Patterns must be a StringType, EOF, TIMEOUT, SRE_Pattern, or a list of + those. Patterns may also be None which results in an empty list (you + might do this if waiting for an EOF or TIMEOUT condition without + expecting any pattern). + + This is used by expect() when calling expect_list(). Thus expect() is + nothing more than:: + + cpl = self.compile_pattern_list(pl) + return self.expect_list(cpl, timeout) + + If you are using expect() within a loop it may be more + efficient to compile the patterns first and then call expect_list(). + This avoid calls in a loop to compile_pattern_list():: + + cpl = self.compile_pattern_list(my_pattern) + while some_condition: + ... + i = self.expect_list(cpl, timeout) + ... + ''' + + if patterns is None: + return [] + if not isinstance(patterns, list): + patterns = [patterns] + + # Allow dot to match \n + compile_flags = re.DOTALL + if self.ignorecase: + compile_flags = compile_flags | re.IGNORECASE + compiled_pattern_list = [] + for idx, p in enumerate(patterns): + if isinstance(p, self.allowed_string_types): + p = self._coerce_expect_string(p) + compiled_pattern_list.append(re.compile(p, compile_flags)) + elif p is EOF: + compiled_pattern_list.append(EOF) + elif p is TIMEOUT: + compiled_pattern_list.append(TIMEOUT) + elif isinstance(p, type(re.compile(''))): + p = self._coerce_expect_re(p) + compiled_pattern_list.append(p) + else: + self._pattern_type_err(p) + return compiled_pattern_list + + def expect(self, pattern, timeout=-1, searchwindowsize=-1, async_=False, **kw): + '''This seeks through the stream until a pattern is matched. The + pattern is overloaded and may take several types. The pattern can be a + StringType, EOF, a compiled re, or a list of any of those types. + Strings will be compiled to re types. This returns the index into the + pattern list. If the pattern was not a list this returns index 0 on a + successful match. This may raise exceptions for EOF or TIMEOUT. To + avoid the EOF or TIMEOUT exceptions add EOF or TIMEOUT to the pattern + list. That will cause expect to match an EOF or TIMEOUT condition + instead of raising an exception. + + If you pass a list of patterns and more than one matches, the first + match in the stream is chosen. If more than one pattern matches at that + point, the leftmost in the pattern list is chosen. For example:: + + # the input is 'foobar' + index = p.expect(['bar', 'foo', 'foobar']) + # returns 1('foo') even though 'foobar' is a "better" match + + Please note, however, that buffering can affect this behavior, since + input arrives in unpredictable chunks. For example:: + + # the input is 'foobar' + index = p.expect(['foobar', 'foo']) + # returns 0('foobar') if all input is available at once, + # but returns 1('foo') if parts of the final 'bar' arrive late + + When a match is found for the given pattern, the class instance + attribute *match* becomes an re.MatchObject result. Should an EOF + or TIMEOUT pattern match, then the match attribute will be an instance + of that exception class. The pairing before and after class + instance attributes are views of the data preceding and following + the matching pattern. On general exception, class attribute + *before* is all data received up to the exception, while *match* and + *after* attributes are value None. + + When the keyword argument timeout is -1 (default), then TIMEOUT will + raise after the default value specified by the class timeout + attribute. When None, TIMEOUT will not be raised and may block + indefinitely until match. + + When the keyword argument searchwindowsize is -1 (default), then the + value specified by the class maxread attribute is used. + + A list entry may be EOF or TIMEOUT instead of a string. This will + catch these exceptions and return the index of the list entry instead + of raising the exception. The attribute 'after' will be set to the + exception type. The attribute 'match' will be None. This allows you to + write code like this:: + + index = p.expect(['good', 'bad', pexpect.EOF, pexpect.TIMEOUT]) + if index == 0: + do_something() + elif index == 1: + do_something_else() + elif index == 2: + do_some_other_thing() + elif index == 3: + do_something_completely_different() + + instead of code like this:: + + try: + index = p.expect(['good', 'bad']) + if index == 0: + do_something() + elif index == 1: + do_something_else() + except EOF: + do_some_other_thing() + except TIMEOUT: + do_something_completely_different() + + These two forms are equivalent. It all depends on what you want. You + can also just expect the EOF if you are waiting for all output of a + child to finish. For example:: + + p = pexpect.spawn('/bin/ls') + p.expect(pexpect.EOF) + print p.before + + If you are trying to optimize for speed then see expect_list(). + + On Python 3.4, or Python 3.3 with asyncio installed, passing + ``async_=True`` will make this return an :mod:`asyncio` coroutine, + which you can yield from to get the same result that this method would + normally give directly. So, inside a coroutine, you can replace this code:: + + index = p.expect(patterns) + + With this non-blocking form:: + + index = yield from p.expect(patterns, async_=True) + ''' + if 'async' in kw: + async_ = kw.pop('async') + if kw: + raise TypeError("Unknown keyword arguments: {}".format(kw)) + + compiled_pattern_list = self.compile_pattern_list(pattern) + return self.expect_list(compiled_pattern_list, + timeout, searchwindowsize, async_) + + def expect_list(self, pattern_list, timeout=-1, searchwindowsize=-1, + async_=False, **kw): + '''This takes a list of compiled regular expressions and returns the + index into the pattern_list that matched the child output. The list may + also contain EOF or TIMEOUT(which are not compiled regular + expressions). This method is similar to the expect() method except that + expect_list() does not recompile the pattern list on every call. This + may help if you are trying to optimize for speed, otherwise just use + the expect() method. This is called by expect(). + + + Like :meth:`expect`, passing ``async_=True`` will make this return an + asyncio coroutine. + ''' + if timeout == -1: + timeout = self.timeout + if 'async' in kw: + async_ = kw.pop('async') + if kw: + raise TypeError("Unknown keyword arguments: {}".format(kw)) + + exp = Expecter(self, searcher_re(pattern_list), searchwindowsize) + if async_: + from ._async import expect_async + return expect_async(exp, timeout) + else: + return exp.expect_loop(timeout) + + def expect_exact(self, pattern_list, timeout=-1, searchwindowsize=-1, + async_=False, **kw): + + '''This is similar to expect(), but uses plain string matching instead + of compiled regular expressions in 'pattern_list'. The 'pattern_list' + may be a string; a list or other sequence of strings; or TIMEOUT and + EOF. + + This call might be faster than expect() for two reasons: string + searching is faster than RE matching and it is possible to limit the + search to just the end of the input buffer. + + This method is also useful when you don't want to have to worry about + escaping regular expression characters that you want to match. + + Like :meth:`expect`, passing ``async_=True`` will make this return an + asyncio coroutine. + ''' + if timeout == -1: + timeout = self.timeout + if 'async' in kw: + async_ = kw.pop('async') + if kw: + raise TypeError("Unknown keyword arguments: {}".format(kw)) + + if (isinstance(pattern_list, self.allowed_string_types) or + pattern_list in (TIMEOUT, EOF)): + pattern_list = [pattern_list] + + def prepare_pattern(pattern): + if pattern in (TIMEOUT, EOF): + return pattern + if isinstance(pattern, self.allowed_string_types): + return self._coerce_expect_string(pattern) + self._pattern_type_err(pattern) + + try: + pattern_list = iter(pattern_list) + except TypeError: + self._pattern_type_err(pattern_list) + pattern_list = [prepare_pattern(p) for p in pattern_list] + + exp = Expecter(self, searcher_string(pattern_list), searchwindowsize) + if async_: + from ._async import expect_async + return expect_async(exp, timeout) + else: + return exp.expect_loop(timeout) + + def expect_loop(self, searcher, timeout=-1, searchwindowsize=-1): + '''This is the common loop used inside expect. The 'searcher' should be + an instance of searcher_re or searcher_string, which describes how and + what to search for in the input. + + See expect() for other arguments, return value and exceptions. ''' + + exp = Expecter(self, searcher, searchwindowsize) + return exp.expect_loop(timeout) + + def read(self, size=-1): + '''This reads at most "size" bytes from the file (less if the read hits + EOF before obtaining size bytes). If the size argument is negative or + omitted, read all data until EOF is reached. The bytes are returned as + a string object. An empty string is returned when EOF is encountered + immediately. ''' + + if size == 0: + return self.string_type() + if size < 0: + # delimiter default is EOF + self.expect(self.delimiter) + return self.before + + # I could have done this more directly by not using expect(), but + # I deliberately decided to couple read() to expect() so that + # I would catch any bugs early and ensure consistent behavior. + # It's a little less efficient, but there is less for me to + # worry about if I have to later modify read() or expect(). + # Note, it's OK if size==-1 in the regex. That just means it + # will never match anything in which case we stop only on EOF. + cre = re.compile(self._coerce_expect_string('.{%d}' % size), re.DOTALL) + # delimiter default is EOF + index = self.expect([cre, self.delimiter]) + if index == 0: + ### FIXME self.before should be ''. Should I assert this? + return self.after + return self.before + + def readline(self, size=-1): + '''This reads and returns one entire line. The newline at the end of + line is returned as part of the string, unless the file ends without a + newline. An empty string is returned if EOF is encountered immediately. + This looks for a newline as a CR/LF pair (\\r\\n) even on UNIX because + this is what the pseudotty device returns. So contrary to what you may + expect you will receive newlines as \\r\\n. + + If the size argument is 0 then an empty string is returned. In all + other cases the size argument is ignored, which is not standard + behavior for a file-like object. ''' + + if size == 0: + return self.string_type() + # delimiter default is EOF + index = self.expect([self.crlf, self.delimiter]) + if index == 0: + return self.before + self.crlf + else: + return self.before + + def __iter__(self): + '''This is to support iterators over a file-like object. + ''' + return iter(self.readline, self.string_type()) + + def readlines(self, sizehint=-1): + '''This reads until EOF using readline() and returns a list containing + the lines thus read. The optional 'sizehint' argument is ignored. + Remember, because this reads until EOF that means the child + process should have closed its stdout. If you run this method on + a child that is still running with its stdout open then this + method will block until it timesout.''' + + lines = [] + while True: + line = self.readline() + if not line: + break + lines.append(line) + return lines + + def fileno(self): + '''Expose file descriptor for a file-like interface + ''' + return self.child_fd + + def flush(self): + '''This does nothing. It is here to support the interface for a + File-like object. ''' + pass + + def isatty(self): + """Overridden in subclass using tty""" + return False + + # For 'with spawn(...) as child:' + def __enter__(self): + return self + + def __exit__(self, etype, evalue, tb): + # We rely on subclasses to implement close(). If they don't, it's not + # clear what a context manager should do. + self.close() diff --git a/lib/python3.12/site-packages/pexpect/utils.py b/lib/python3.12/site-packages/pexpect/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..f774519609005dface41fd15021c7f237f091341 --- /dev/null +++ b/lib/python3.12/site-packages/pexpect/utils.py @@ -0,0 +1,187 @@ +import os +import sys +import stat +import select +import time +import errno + +try: + InterruptedError +except NameError: + # Alias Python2 exception to Python3 + InterruptedError = select.error + +if sys.version_info[0] >= 3: + string_types = (str,) +else: + string_types = (unicode, str) + + +def is_executable_file(path): + """Checks that path is an executable regular file, or a symlink towards one. + + This is roughly ``os.path isfile(path) and os.access(path, os.X_OK)``. + """ + # follow symlinks, + fpath = os.path.realpath(path) + + if not os.path.isfile(fpath): + # non-files (directories, fifo, etc.) + return False + + mode = os.stat(fpath).st_mode + + if (sys.platform.startswith('sunos') + and os.getuid() == 0): + # When root on Solaris, os.X_OK is True for *all* files, irregardless + # of their executability -- instead, any permission bit of any user, + # group, or other is fine enough. + # + # (This may be true for other "Unix98" OS's such as HP-UX and AIX) + return bool(mode & (stat.S_IXUSR | + stat.S_IXGRP | + stat.S_IXOTH)) + + return os.access(fpath, os.X_OK) + + +def which(filename, env=None): + '''This takes a given filename; tries to find it in the environment path; + then checks if it is executable. This returns the full path to the filename + if found and executable. Otherwise this returns None.''' + + # Special case where filename contains an explicit path. + if os.path.dirname(filename) != '' and is_executable_file(filename): + return filename + if env is None: + env = os.environ + p = env.get('PATH') + if not p: + p = os.defpath + pathlist = p.split(os.pathsep) + for path in pathlist: + ff = os.path.join(path, filename) + if is_executable_file(ff): + return ff + return None + + +def split_command_line(command_line): + + '''This splits a command line into a list of arguments. It splits arguments + on spaces, but handles embedded quotes, doublequotes, and escaped + characters. It's impossible to do this with a regular expression, so I + wrote a little state machine to parse the command line. ''' + + arg_list = [] + arg = '' + + # Constants to name the states we can be in. + state_basic = 0 + state_esc = 1 + state_singlequote = 2 + state_doublequote = 3 + # The state when consuming whitespace between commands. + state_whitespace = 4 + state = state_basic + + for c in command_line: + if state == state_basic or state == state_whitespace: + if c == '\\': + # Escape the next character + state = state_esc + elif c == r"'": + # Handle single quote + state = state_singlequote + elif c == r'"': + # Handle double quote + state = state_doublequote + elif c.isspace(): + # Add arg to arg_list if we aren't in the middle of whitespace. + if state == state_whitespace: + # Do nothing. + None + else: + arg_list.append(arg) + arg = '' + state = state_whitespace + else: + arg = arg + c + state = state_basic + elif state == state_esc: + arg = arg + c + state = state_basic + elif state == state_singlequote: + if c == r"'": + state = state_basic + else: + arg = arg + c + elif state == state_doublequote: + if c == r'"': + state = state_basic + else: + arg = arg + c + + if arg != '': + arg_list.append(arg) + return arg_list + + +def select_ignore_interrupts(iwtd, owtd, ewtd, timeout=None): + + '''This is a wrapper around select.select() that ignores signals. If + select.select raises a select.error exception and errno is an EINTR + error then it is ignored. Mainly this is used to ignore sigwinch + (terminal resize). ''' + + # if select() is interrupted by a signal (errno==EINTR) then + # we loop back and enter the select() again. + if timeout is not None: + end_time = time.time() + timeout + while True: + try: + return select.select(iwtd, owtd, ewtd, timeout) + except InterruptedError: + err = sys.exc_info()[1] + if err.args[0] == errno.EINTR: + # if we loop back we have to subtract the + # amount of time we already waited. + if timeout is not None: + timeout = end_time - time.time() + if timeout < 0: + return([], [], []) + else: + # something else caused the select.error, so + # this actually is an exception. + raise + + +def poll_ignore_interrupts(fds, timeout=None): + '''Simple wrapper around poll to register file descriptors and + ignore signals.''' + + if timeout is not None: + end_time = time.time() + timeout + + poller = select.poll() + for fd in fds: + poller.register(fd, select.POLLIN | select.POLLPRI | select.POLLHUP | select.POLLERR) + + while True: + try: + timeout_ms = None if timeout is None else timeout * 1000 + results = poller.poll(timeout_ms) + return [afd for afd, _ in results] + except InterruptedError: + err = sys.exc_info()[1] + if err.args[0] == errno.EINTR: + # if we loop back we have to subtract the + # amount of time we already waited. + if timeout is not None: + timeout = end_time - time.time() + if timeout < 0: + return [] + else: + # something else caused the select.error, so + # this actually is an exception. + raise diff --git a/lib/python3.12/site-packages/setproctitle/__init__.py b/lib/python3.12/site-packages/setproctitle/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..68cd1578e7430114550c939efd310e591071db79 --- /dev/null +++ b/lib/python3.12/site-packages/setproctitle/__init__.py @@ -0,0 +1,57 @@ +"""Allow customization of the process title.""" + +import os +import sys +import logging + +logger = logging.getLogger("setproctitle") + +__version__ = "1.3.7" + +__all__ = [ + "setproctitle", + "getproctitle", + "setthreadtitle", + "getthreadtitle", +] + + +def setproctitle(title: str) -> None: + logger.debug("setproctitle C module not available") + return None + + +def getproctitle() -> str: + logger.debug("setproctitle C module not available") + return " ".join(sys.argv) + + +def setthreadtitle(title: str) -> None: + logger.debug("setproctitle C module not available") + return None + + +def getthreadtitle() -> str: + logger.debug("setproctitle C module not available") + return "" + + +try: + from . import _setproctitle # type: ignore +except ImportError as e: + # Emulate SPT_DEBUG showing process info in the C module. + if os.environ.get("SPT_DEBUG", ""): + logging.basicConfig() + logger.setLevel(logging.DEBUG) + logger.debug("failed to import setproctitle: %s", e) +else: + setproctitle = _setproctitle.setproctitle # noqa: F811 + getproctitle = _setproctitle.getproctitle # noqa: F811 + setthreadtitle = _setproctitle.setthreadtitle # noqa: F811 + getthreadtitle = _setproctitle.getthreadtitle # noqa: F811 + + +# Call getproctitle to initialize structures and avoid problems caused +# by fork() on macOS (see #113). +if sys.platform == "darwin": + getproctitle() diff --git a/lib/python3.12/site-packages/setproctitle/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/setproctitle/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..dc74ac05bf301afd7ff3a6fcaec22a49576127e3 Binary files /dev/null and b/lib/python3.12/site-packages/setproctitle/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/setproctitle/_setproctitle.cpython-312-x86_64-linux-gnu.so b/lib/python3.12/site-packages/setproctitle/_setproctitle.cpython-312-x86_64-linux-gnu.so new file mode 100644 index 0000000000000000000000000000000000000000..da678b7bf012ae636aeeb3b397f1db3c6f945d92 Binary files /dev/null and b/lib/python3.12/site-packages/setproctitle/_setproctitle.cpython-312-x86_64-linux-gnu.so differ diff --git a/lib/python3.12/site-packages/setproctitle/py.typed b/lib/python3.12/site-packages/setproctitle/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tiktoken/__init__.py b/lib/python3.12/site-packages/tiktoken/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..f90afc7bec5ec240018a227adb45a053657808c3 --- /dev/null +++ b/lib/python3.12/site-packages/tiktoken/__init__.py @@ -0,0 +1,8 @@ +# This is the public API of tiktoken +from .core import Encoding as Encoding +from .model import encoding_for_model as encoding_for_model +from .model import encoding_name_for_model as encoding_name_for_model +from .registry import get_encoding as get_encoding +from .registry import list_encoding_names as list_encoding_names + +__version__ = "0.12.0" diff --git a/lib/python3.12/site-packages/tiktoken/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/tiktoken/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..08dbed24a13ec5a6f0bbf4eb385791157f2ab0f8 Binary files /dev/null and b/lib/python3.12/site-packages/tiktoken/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tiktoken/__pycache__/_educational.cpython-312.pyc b/lib/python3.12/site-packages/tiktoken/__pycache__/_educational.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b176825190bd8916ac971d8db84a112c3e570c08 Binary files /dev/null and b/lib/python3.12/site-packages/tiktoken/__pycache__/_educational.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tiktoken/__pycache__/core.cpython-312.pyc b/lib/python3.12/site-packages/tiktoken/__pycache__/core.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7ee80776325717ec6e0f550e467892fae0ec3543 Binary files /dev/null and b/lib/python3.12/site-packages/tiktoken/__pycache__/core.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tiktoken/__pycache__/load.cpython-312.pyc b/lib/python3.12/site-packages/tiktoken/__pycache__/load.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..94cb9710ca29ff59393469c0f72c4560bed3b9a9 Binary files /dev/null and b/lib/python3.12/site-packages/tiktoken/__pycache__/load.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tiktoken/__pycache__/model.cpython-312.pyc b/lib/python3.12/site-packages/tiktoken/__pycache__/model.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..939a05e1ad514887ef4f704444a5251749f525f9 Binary files /dev/null and b/lib/python3.12/site-packages/tiktoken/__pycache__/model.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tiktoken/__pycache__/registry.cpython-312.pyc b/lib/python3.12/site-packages/tiktoken/__pycache__/registry.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..6d3b9bccfd40a550647d728c5ba7c3f74c56233d Binary files /dev/null and b/lib/python3.12/site-packages/tiktoken/__pycache__/registry.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/tiktoken/_educational.py b/lib/python3.12/site-packages/tiktoken/_educational.py new file mode 100644 index 0000000000000000000000000000000000000000..317e77560c2b95c2c8a7b3d0d170b2af6de239c9 --- /dev/null +++ b/lib/python3.12/site-packages/tiktoken/_educational.py @@ -0,0 +1,223 @@ +"""This is an educational implementation of the byte pair encoding algorithm.""" + +from __future__ import annotations + +import collections + +import regex + +import tiktoken + + +class SimpleBytePairEncoding: + def __init__(self, *, pat_str: str, mergeable_ranks: dict[bytes, int]) -> None: + """Creates an Encoding object.""" + # A regex pattern string that is used to split the input text + self.pat_str = pat_str + # A dictionary mapping token bytes to their ranks. The ranks correspond to merge priority + self.mergeable_ranks = mergeable_ranks + + self._decoder = {token: token_bytes for token_bytes, token in mergeable_ranks.items()} + self._pat = regex.compile(pat_str) + + def encode(self, text: str, visualise: str | None = "colour") -> list[int]: + """Encodes a string into tokens. + + >>> enc.encode("hello world") + [388, 372] + """ + # Use the regex to split the text into (approximately) words + words = self._pat.findall(text) + tokens = [] + for word in words: + # Turn each word into tokens, using the byte pair encoding algorithm + word_bytes = word.encode("utf-8") + word_tokens = bpe_encode(self.mergeable_ranks, word_bytes, visualise=visualise) + tokens.extend(word_tokens) + return tokens + + def decode_bytes(self, tokens: list[int]) -> bytes: + """Decodes a list of tokens into bytes. + + >>> enc.decode_bytes([388, 372]) + b'hello world' + """ + return b"".join(self._decoder[token] for token in tokens) + + def decode(self, tokens: list[int]) -> str: + """Decodes a list of tokens into a string. + + Decoded bytes are not guaranteed to be valid UTF-8. In that case, we replace + the invalid bytes with the replacement character "�". + + >>> enc.decode([388, 372]) + 'hello world' + """ + return self.decode_bytes(tokens).decode("utf-8", errors="replace") + + def decode_tokens_bytes(self, tokens: list[int]) -> list[bytes]: + """Decodes a list of tokens into a list of bytes. + + Useful for visualising how a string is tokenised. + + >>> enc.decode_tokens_bytes([388, 372]) + [b'hello', b' world'] + """ + return [self._decoder[token] for token in tokens] + + @staticmethod + def train(training_data: str, vocab_size: int, pat_str: str): + """Train a BPE tokeniser on some data!""" + mergeable_ranks = bpe_train(data=training_data, vocab_size=vocab_size, pat_str=pat_str) + return SimpleBytePairEncoding(pat_str=pat_str, mergeable_ranks=mergeable_ranks) + + @staticmethod + def from_tiktoken(encoding): + if isinstance(encoding, str): + encoding = tiktoken.get_encoding(encoding) + return SimpleBytePairEncoding( + pat_str=encoding._pat_str, mergeable_ranks=encoding._mergeable_ranks + ) + + +def bpe_encode( + mergeable_ranks: dict[bytes, int], input: bytes, visualise: str | None = "colour" +) -> list[int]: + parts = [bytes([b]) for b in input] + while True: + # See the intermediate merges play out! + if visualise: + if visualise in ["colour", "color"]: + visualise_tokens(parts) + elif visualise == "simple": + print(parts) + + # Iterate over all pairs and find the pair we want to merge the most + min_idx = None + min_rank = None + for i, pair in enumerate(zip(parts[:-1], parts[1:])): + rank = mergeable_ranks.get(pair[0] + pair[1]) + if rank is not None and (min_rank is None or rank < min_rank): + min_idx = i + min_rank = rank + + # If there were no pairs we could merge, we're done! + if min_rank is None: + break + assert min_idx is not None + + # Otherwise, merge that pair and leave the rest unchanged. Then repeat. + parts = parts[:min_idx] + [parts[min_idx] + parts[min_idx + 1]] + parts[min_idx + 2 :] + + if visualise: + print() + + tokens = [mergeable_ranks[part] for part in parts] + return tokens + + +def bpe_train( + data: str, vocab_size: int, pat_str: str, visualise: str | None = "colour" +) -> dict[bytes, int]: + # First, add tokens for each individual byte value + if vocab_size < 2**8: + raise ValueError("vocab_size must be at least 256, so we can encode all bytes") + ranks = {} + for i in range(2**8): + ranks[bytes([i])] = i + + # Splinter up our data into lists of bytes + # data = "Hello world" + # words = [ + # [b'H', b'e', b'l', b'l', b'o'], + # [b' ', b'w', b'o', b'r', b'l', b'd'] + # ] + words: list[list[bytes]] = [ + [bytes([b]) for b in word.encode("utf-8")] for word in regex.findall(pat_str, data) + ] + + # Now, use our data to figure out which merges we should make + while len(ranks) < vocab_size: + # Find the most common pair. This will become our next token + stats = collections.Counter() + for piece in words: + for pair in zip(piece[:-1], piece[1:]): + stats[pair] += 1 + + most_common_pair = max(stats, key=lambda x: stats[x]) + token_bytes = most_common_pair[0] + most_common_pair[1] + token = len(ranks) + # Add the new token! + ranks[token_bytes] = token + + # Now merge that most common pair in all the words. That is, update our training data + # to reflect our decision to make that pair into a new token. + new_words = [] + for word in words: + new_word = [] + i = 0 + while i < len(word) - 1: + if (word[i], word[i + 1]) == most_common_pair: + # We found our pair! Merge it + new_word.append(token_bytes) + i += 2 + else: + new_word.append(word[i]) + i += 1 + if i == len(word) - 1: + new_word.append(word[i]) + new_words.append(new_word) + words = new_words + + # See the intermediate merges play out! + if visualise: + print(f"The current most common pair is {most_common_pair[0]} + {most_common_pair[1]}") + print(f"So we made {token_bytes} our {len(ranks)}th token") + if visualise in ["colour", "color"]: + print("Now the first fifty words in our training data look like:") + visualise_tokens([token for word in words[:50] for token in word]) + elif visualise == "simple": + print("Now the first twenty words in our training data look like:") + for word in words[:20]: + print(word) + print("\n") + + return ranks + + +def visualise_tokens(token_values: list[bytes]) -> None: + background = [f"\u001b[48;5;{i}m" for i in [167, 179, 185, 77, 80, 68, 134]] + # If token boundaries do not occur at unicode character boundaries, it's unclear how best to + # visualise the token. Here, we'll just use the unicode replacement character to represent some + # fraction of a character. + unicode_token_values = [x.decode("utf-8", errors="replace") for x in token_values] + + running_length = 0 + last_color = None + for token in unicode_token_values: + color = background[running_length % len(background)] + if color == last_color: + color = background[(running_length + 1) % len(background)] + assert color != last_color + last_color = color + running_length += len(token) + print(color + token, end="") + print("\u001b[0m") + + +def train_simple_encoding(): + gpt2_pattern = ( + r"""'s|'t|'re|'ve|'m|'ll|'d| ?[\p{L}]+| ?[\p{N}]+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+""" + ) + with open(__file__) as f: + data = f.read() + + enc = SimpleBytePairEncoding.train(data, vocab_size=600, pat_str=gpt2_pattern) + + print("This is the sequence of merges performed in order to encode 'hello world':") + tokens = enc.encode("hello world") + assert enc.decode(tokens) == "hello world" + assert enc.decode_bytes(tokens) == b"hello world" + assert enc.decode_tokens_bytes(tokens) == [b"hello", b" world"] + + return enc diff --git a/lib/python3.12/site-packages/tiktoken/core.py b/lib/python3.12/site-packages/tiktoken/core.py new file mode 100644 index 0000000000000000000000000000000000000000..225fffb87d666004022ee61b577d0bc8ceeba5a8 --- /dev/null +++ b/lib/python3.12/site-packages/tiktoken/core.py @@ -0,0 +1,446 @@ +from __future__ import annotations + +import functools +from concurrent.futures import ThreadPoolExecutor +from typing import TYPE_CHECKING, AbstractSet, Collection, Literal, NoReturn, Sequence + +from tiktoken import _tiktoken + +if TYPE_CHECKING: + import re + + import numpy as np + import numpy.typing as npt + + +class Encoding: + def __init__( + self, + name: str, + *, + pat_str: str, + mergeable_ranks: dict[bytes, int], + special_tokens: dict[str, int], + explicit_n_vocab: int | None = None, + ): + """Creates an Encoding object. + + See openai_public.py for examples of how to construct an Encoding object. + + Args: + name: The name of the encoding. It should be clear from the name of the encoding + what behaviour to expect, in particular, encodings with different special tokens + should have different names. + pat_str: A regex pattern string that is used to split the input text. + mergeable_ranks: A dictionary mapping mergeable token bytes to their ranks. The ranks + must correspond to merge priority. + special_tokens: A dictionary mapping special token strings to their token values. + explicit_n_vocab: The number of tokens in the vocabulary. If provided, it is checked + that the number of mergeable tokens and special tokens is equal to this number. + """ + self.name = name + + self._pat_str = pat_str + self._mergeable_ranks = mergeable_ranks + self._special_tokens = special_tokens + + self.max_token_value = max( + max(mergeable_ranks.values()), max(special_tokens.values(), default=0) + ) + if explicit_n_vocab: + assert len(mergeable_ranks) + len(special_tokens) == explicit_n_vocab + assert self.max_token_value == explicit_n_vocab - 1 + + self._core_bpe = _tiktoken.CoreBPE(mergeable_ranks, special_tokens, pat_str) + + def __repr__(self) -> str: + return f"" + + # ==================== + # Encoding + # ==================== + + def encode_ordinary(self, text: str) -> list[int]: + """Encodes a string into tokens, ignoring special tokens. + + This is equivalent to `encode(text, disallowed_special=())` (but slightly faster). + + ``` + >>> enc.encode_ordinary("hello world") + [31373, 995] + """ + try: + return self._core_bpe.encode_ordinary(text) + except UnicodeEncodeError: + # See comment in encode + text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace") + return self._core_bpe.encode_ordinary(text) + + def encode( + self, + text: str, + *, + allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006 + disallowed_special: Literal["all"] | Collection[str] = "all", + ) -> list[int]: + """Encodes a string into tokens. + + Special tokens are artificial tokens used to unlock capabilities from a model, + such as fill-in-the-middle. So we want to be careful about accidentally encoding special + tokens, since they can be used to trick a model into doing something we don't want it to do. + + Hence, by default, encode will raise an error if it encounters text that corresponds + to a special token. This can be controlled on a per-token level using the `allowed_special` + and `disallowed_special` parameters. In particular: + - Setting `disallowed_special` to () will prevent this function from raising errors and + cause all text corresponding to special tokens to be encoded as natural text. + - Setting `allowed_special` to "all" will cause this function to treat all text + corresponding to special tokens to be encoded as special tokens. + + ``` + >>> enc.encode("hello world") + [31373, 995] + >>> enc.encode("<|endoftext|>", allowed_special={"<|endoftext|>"}) + [50256] + >>> enc.encode("<|endoftext|>", allowed_special="all") + [50256] + >>> enc.encode("<|endoftext|>") + # Raises ValueError + >>> enc.encode("<|endoftext|>", disallowed_special=()) + [27, 91, 437, 1659, 5239, 91, 29] + ``` + """ + if allowed_special == "all": + allowed_special = self.special_tokens_set + if disallowed_special == "all": + disallowed_special = self.special_tokens_set - allowed_special + if disallowed_special: + if not isinstance(disallowed_special, frozenset): + disallowed_special = frozenset(disallowed_special) + if match := _special_token_regex(disallowed_special).search(text): + raise_disallowed_special_token(match.group()) + + try: + return self._core_bpe.encode(text, allowed_special) + except UnicodeEncodeError: + # BPE operates on bytes, but the regex operates on unicode. If we pass a str that is + # invalid UTF-8 to Rust, it will rightfully complain. Here we do a quick and dirty + # fixup for any surrogate pairs that may have sneaked their way into the text. + # Technically, this introduces a place where encode + decode doesn't roundtrip a Python + # string, but given that this is input we want to support, maybe that's okay. + # Also we use errors="replace" to handle weird things like lone surrogates. + text = text.encode("utf-16", "surrogatepass").decode("utf-16", "replace") + return self._core_bpe.encode(text, allowed_special) + + def encode_to_numpy( + self, + text: str, + *, + allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006 + disallowed_special: Literal["all"] | Collection[str] = "all", + ) -> npt.NDArray[np.uint32]: + """Encodes a string into tokens, returning a numpy array. + + Avoids the overhead of copying the token buffer into a Python list. + """ + if allowed_special == "all": + allowed_special = self.special_tokens_set + if disallowed_special == "all": + disallowed_special = self.special_tokens_set - allowed_special + if disallowed_special: + if not isinstance(disallowed_special, frozenset): + disallowed_special = frozenset(disallowed_special) + if match := _special_token_regex(disallowed_special).search(text): + raise_disallowed_special_token(match.group()) + + import numpy as np + + buffer = self._core_bpe.encode_to_tiktoken_buffer(text, allowed_special) + return np.frombuffer(buffer, dtype=np.uint32) + + def encode_ordinary_batch(self, text: list[str], *, num_threads: int = 8) -> list[list[int]]: + """Encodes a list of strings into tokens, in parallel, ignoring special tokens. + + This is equivalent to `encode_batch(text, disallowed_special=())` (but slightly faster). + + ``` + >>> enc.encode_ordinary_batch(["hello world", "goodbye world"]) + [[31373, 995], [11274, 16390, 995]] + ``` + """ + encoder = functools.partial(self.encode_ordinary) + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(encoder, text)) + + def encode_batch( + self, + text: list[str], + *, + num_threads: int = 8, + allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006 + disallowed_special: Literal["all"] | Collection[str] = "all", + ) -> list[list[int]]: + """Encodes a list of strings into tokens, in parallel. + + See `encode` for more details on `allowed_special` and `disallowed_special`. + + ``` + >>> enc.encode_batch(["hello world", "goodbye world"]) + [[31373, 995], [11274, 16390, 995]] + ``` + """ + if allowed_special == "all": + allowed_special = self.special_tokens_set + if disallowed_special == "all": + disallowed_special = self.special_tokens_set - allowed_special + if not isinstance(disallowed_special, frozenset): + disallowed_special = frozenset(disallowed_special) + + encoder = functools.partial( + self.encode, allowed_special=allowed_special, disallowed_special=disallowed_special + ) + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(encoder, text)) + + def encode_with_unstable( + self, + text: str, + *, + allowed_special: Literal["all"] | AbstractSet[str] = set(), # noqa: B006 + disallowed_special: Literal["all"] | Collection[str] = "all", + ) -> tuple[list[int], list[list[int]]]: + """Encodes a string into stable tokens and possible completion sequences. + + Note that the stable tokens will only represent a substring of `text`. + + See `encode` for more details on `allowed_special` and `disallowed_special`. + + This API should itself be considered unstable. + + ``` + >>> enc.encode_with_unstable("hello fanta") + ([31373], [(277, 4910), (5113, 265), ..., (8842,)]) + + >>> text = "..." + >>> stable_tokens, completions = enc.encode_with_unstable(text) + >>> assert text.encode().startswith(enc.decode_bytes(stable_tokens)) + >>> assert all(enc.decode_bytes(stable_tokens + seq).startswith(text.encode()) for seq in completions) + ``` + """ + if allowed_special == "all": + allowed_special = self.special_tokens_set + if disallowed_special == "all": + disallowed_special = self.special_tokens_set - allowed_special + if disallowed_special: + if not isinstance(disallowed_special, frozenset): + disallowed_special = frozenset(disallowed_special) + if match := _special_token_regex(disallowed_special).search(text): + raise_disallowed_special_token(match.group()) + + return self._core_bpe.encode_with_unstable(text, allowed_special) + + def encode_single_token(self, text_or_bytes: str | bytes) -> int: + """Encodes text corresponding to a single token to its token value. + + NOTE: this will encode all special tokens. + + Raises `KeyError` if the token is not in the vocabulary. + + ``` + >>> enc.encode_single_token("hello") + 31373 + ``` + """ + if isinstance(text_or_bytes, str): + text_or_bytes = text_or_bytes.encode("utf-8") + return self._core_bpe.encode_single_token(text_or_bytes) + + # ==================== + # Decoding + # ==================== + + def decode_bytes(self, tokens: Sequence[int]) -> bytes: + """Decodes a list of tokens into bytes. + + ``` + >>> enc.decode_bytes([31373, 995]) + b'hello world' + ``` + """ + return self._core_bpe.decode_bytes(tokens) + + def decode(self, tokens: Sequence[int], errors: str = "replace") -> str: + """Decodes a list of tokens into a string. + + WARNING: the default behaviour of this function is lossy, since decoded bytes are not + guaranteed to be valid UTF-8. You can control this behaviour using the `errors` parameter, + for instance, setting `errors=strict`. + + ``` + >>> enc.decode([31373, 995]) + 'hello world' + ``` + """ + return self._core_bpe.decode_bytes(tokens).decode("utf-8", errors=errors) + + def decode_single_token_bytes(self, token: int) -> bytes: + """Decodes a token into bytes. + + NOTE: this will decode all special tokens. + + Raises `KeyError` if the token is not in the vocabulary. + + ``` + >>> enc.decode_single_token_bytes(31373) + b'hello' + ``` + """ + return self._core_bpe.decode_single_token_bytes(token) + + def decode_tokens_bytes(self, tokens: Sequence[int]) -> list[bytes]: + """Decodes a list of tokens into a list of bytes. + + Useful for visualising tokenisation. + >>> enc.decode_tokens_bytes([31373, 995]) + [b'hello', b' world'] + """ + return [self.decode_single_token_bytes(token) for token in tokens] + + def decode_with_offsets(self, tokens: Sequence[int]) -> tuple[str, list[int]]: + """Decodes a list of tokens into a string and a list of offsets. + + Each offset is the index into text corresponding to the start of each token. + If UTF-8 character boundaries do not line up with token boundaries, the offset is the index + of the first character that contains bytes from the token. + + This will currently raise if given tokens that decode to invalid UTF-8; this behaviour may + change in the future to be more permissive. + + >>> enc.decode_with_offsets([31373, 995]) + ('hello world', [0, 5]) + """ + token_bytes = self.decode_tokens_bytes(tokens) + + text_len = 0 + offsets = [] + for token in token_bytes: + offsets.append(max(0, text_len - (0x80 <= token[0] < 0xC0))) + text_len += sum(1 for c in token if not 0x80 <= c < 0xC0) + + # TODO: assess correctness for errors="ignore" and errors="replace" + text = b"".join(token_bytes).decode("utf-8", errors="strict") + return text, offsets + + def decode_batch( + self, batch: Sequence[Sequence[int]], *, errors: str = "replace", num_threads: int = 8 + ) -> list[str]: + """Decodes a batch (list of lists of tokens) into a list of strings.""" + decoder = functools.partial(self.decode, errors=errors) + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(decoder, batch)) + + def decode_bytes_batch( + self, batch: Sequence[Sequence[int]], *, num_threads: int = 8 + ) -> list[bytes]: + """Decodes a batch (list of lists of tokens) into a list of bytes.""" + with ThreadPoolExecutor(num_threads) as e: + return list(e.map(self.decode_bytes, batch)) + + # ==================== + # Miscellaneous + # ==================== + + def token_byte_values(self) -> list[bytes]: + """Returns the list of all token byte values.""" + return self._core_bpe.token_byte_values() + + @property + def eot_token(self) -> int: + return self._special_tokens["<|endoftext|>"] + + @functools.cached_property + def special_tokens_set(self) -> set[str]: + return set(self._special_tokens.keys()) + + def is_special_token(self, token: int) -> bool: + assert isinstance(token, int) + return token in self._special_token_values + + @property + def n_vocab(self) -> int: + """For backwards compatibility. Prefer to use `enc.max_token_value + 1`.""" + return self.max_token_value + 1 + + # ==================== + # Private + # ==================== + + def _encode_single_piece(self, text_or_bytes: str | bytes) -> list[int]: + """Encodes text corresponding to bytes without a regex split. + + NOTE: this will not encode any special tokens. + + ``` + >>> enc.encode_single_piece("helloqqqq") + [31373, 38227, 38227] + ``` + """ + if isinstance(text_or_bytes, str): + text_or_bytes = text_or_bytes.encode("utf-8") + return self._core_bpe.encode_single_piece(text_or_bytes) + + def _encode_only_native_bpe(self, text: str) -> list[int]: + """Encodes a string into tokens, but do regex splitting in Python.""" + # We need specifically `regex` in order to compile pat_str due to e.g. \p + import regex + + _unused_pat = regex.compile(self._pat_str) + ret = [] + for piece in regex.findall(_unused_pat, text): + ret.extend(self._core_bpe.encode_single_piece(piece.encode("utf-8"))) + return ret + + def _encode_bytes(self, text: bytes) -> list[int]: + return self._core_bpe._encode_bytes(text) + + def __getstate__(self) -> object: + import tiktoken.registry + + # As an optimisation, pickle registered encodings by reference + if self is tiktoken.registry.ENCODINGS.get(self.name): + return self.name + return { + "name": self.name, + "pat_str": self._pat_str, + "mergeable_ranks": self._mergeable_ranks, + "special_tokens": self._special_tokens, + } + + def __setstate__(self, value: object) -> None: + import tiktoken.registry + + if isinstance(value, str): + self.__dict__ = tiktoken.registry.get_encoding(value).__dict__ + return + self.__init__(**value) + + +@functools.lru_cache(maxsize=128) +def _special_token_regex(tokens: frozenset[str]) -> re.Pattern[str]: + try: + import regex as re + except ImportError: + import re + inner = "|".join(re.escape(token) for token in tokens) + return re.compile(f"({inner})") + + +def raise_disallowed_special_token(token: str) -> NoReturn: + raise ValueError( + f"Encountered text corresponding to disallowed special token {token!r}.\n" + "If you want this text to be encoded as a special token, " + f"pass it to `allowed_special`, e.g. `allowed_special={{{token!r}, ...}}`.\n" + f"If you want this text to be encoded as normal text, disable the check for this token " + f"by passing `disallowed_special=(enc.special_tokens_set - {{{token!r}}})`.\n" + "To disable this check for all special tokens, pass `disallowed_special=()`.\n" + ) diff --git a/lib/python3.12/site-packages/tiktoken/load.py b/lib/python3.12/site-packages/tiktoken/load.py new file mode 100644 index 0000000000000000000000000000000000000000..dc2eba6699d84489eb1e4aa1d1d13cc82910e801 --- /dev/null +++ b/lib/python3.12/site-packages/tiktoken/load.py @@ -0,0 +1,172 @@ +from __future__ import annotations + +import base64 +import hashlib +import os + + +def read_file(blobpath: str) -> bytes: + if "://" not in blobpath: + with open(blobpath, "rb", buffering=0) as f: + return f.read() + + if blobpath.startswith(("http://", "https://")): + # avoiding blobfile for public files helps avoid auth issues, like MFA prompts. + import requests + + resp = requests.get(blobpath) + resp.raise_for_status() + return resp.content + + try: + import blobfile + except ImportError as e: + raise ImportError( + "blobfile is not installed. Please install it by running `pip install blobfile`." + ) from e + with blobfile.BlobFile(blobpath, "rb") as f: + return f.read() + + +def check_hash(data: bytes, expected_hash: str) -> bool: + actual_hash = hashlib.sha256(data).hexdigest() + return actual_hash == expected_hash + + +def read_file_cached(blobpath: str, expected_hash: str | None = None) -> bytes: + user_specified_cache = True + if "TIKTOKEN_CACHE_DIR" in os.environ: + cache_dir = os.environ["TIKTOKEN_CACHE_DIR"] + elif "DATA_GYM_CACHE_DIR" in os.environ: + cache_dir = os.environ["DATA_GYM_CACHE_DIR"] + else: + import tempfile + + cache_dir = os.path.join(tempfile.gettempdir(), "data-gym-cache") + user_specified_cache = False + + if cache_dir == "": + # disable caching + return read_file(blobpath) + + cache_key = hashlib.sha1(blobpath.encode()).hexdigest() + + cache_path = os.path.join(cache_dir, cache_key) + if os.path.exists(cache_path): + with open(cache_path, "rb", buffering=0) as f: + data = f.read() + if expected_hash is None or check_hash(data, expected_hash): + return data + + # the cached file does not match the hash, remove it and re-fetch + try: + os.remove(cache_path) + except OSError: + pass + + contents = read_file(blobpath) + if expected_hash and not check_hash(contents, expected_hash): + raise ValueError( + f"Hash mismatch for data downloaded from {blobpath} (expected {expected_hash}). " + f"This may indicate a corrupted download. Please try again." + ) + + import uuid + + try: + os.makedirs(cache_dir, exist_ok=True) + tmp_filename = cache_path + "." + str(uuid.uuid4()) + ".tmp" + with open(tmp_filename, "wb") as f: + f.write(contents) + os.rename(tmp_filename, cache_path) + except OSError: + # don't raise if we can't write to the default cache, e.g. issue #75 + if user_specified_cache: + raise + + return contents + + +def data_gym_to_mergeable_bpe_ranks( + vocab_bpe_file: str, + encoder_json_file: str, + vocab_bpe_hash: str | None = None, + encoder_json_hash: str | None = None, + clobber_one_byte_tokens: bool = False, +) -> dict[bytes, int]: + # NB: do not add caching to this function + rank_to_intbyte = [b for b in range(2**8) if chr(b).isprintable() and chr(b) != " "] + + data_gym_byte_to_byte = {chr(b): b for b in rank_to_intbyte} + n = 0 + for b in range(2**8): + if b not in rank_to_intbyte: + rank_to_intbyte.append(b) + data_gym_byte_to_byte[chr(2**8 + n)] = b + n += 1 + assert len(rank_to_intbyte) == 2**8 + + # vocab_bpe contains the merges along with associated ranks + vocab_bpe_contents = read_file_cached(vocab_bpe_file, vocab_bpe_hash).decode() + bpe_merges = [tuple(merge_str.split()) for merge_str in vocab_bpe_contents.split("\n")[1:-1]] + + def decode_data_gym(value: str) -> bytes: + return bytes(data_gym_byte_to_byte[b] for b in value) + + # add the single byte tokens + # if clobber_one_byte_tokens is True, we'll replace these with ones from the encoder json + bpe_ranks = {bytes([b]): i for i, b in enumerate(rank_to_intbyte)} + del rank_to_intbyte + + # add the merged tokens + n = len(bpe_ranks) + for first, second in bpe_merges: + bpe_ranks[decode_data_gym(first) + decode_data_gym(second)] = n + n += 1 + + import json + + # check that the encoder file matches the merges file + # this sanity check is important since tiktoken assumes that ranks are ordered the same + # as merge priority + encoder_json = json.loads(read_file_cached(encoder_json_file, encoder_json_hash)) + encoder_json_loaded = {decode_data_gym(k): v for k, v in encoder_json.items()} + # drop these two special tokens if present, since they're not mergeable bpe tokens + encoder_json_loaded.pop(b"<|endoftext|>", None) + encoder_json_loaded.pop(b"<|startoftext|>", None) + + if clobber_one_byte_tokens: + for k in encoder_json_loaded: + if len(k) == 1: + bpe_ranks[k] = encoder_json_loaded[k] + + assert bpe_ranks == encoder_json_loaded + + return bpe_ranks + + +def dump_tiktoken_bpe(bpe_ranks: dict[bytes, int], tiktoken_bpe_file: str) -> None: + try: + import blobfile + except ImportError as e: + raise ImportError( + "blobfile is not installed. Please install it by running `pip install blobfile`." + ) from e + with blobfile.BlobFile(tiktoken_bpe_file, "wb") as f: + for token, rank in sorted(bpe_ranks.items(), key=lambda x: x[1]): + f.write(base64.b64encode(token) + b" " + str(rank).encode() + b"\n") + + +def load_tiktoken_bpe(tiktoken_bpe_file: str, expected_hash: str | None = None) -> dict[bytes, int]: + # NB: do not add caching to this function + contents = read_file_cached(tiktoken_bpe_file, expected_hash) + ret = {} + for line in contents.splitlines(): + if not line: + continue + try: + token, rank = line.split() + ret[base64.b64decode(token)] = int(rank) + except Exception as e: + raise ValueError(f"Error parsing line {line!r} in {tiktoken_bpe_file}") from e + return ret diff --git a/lib/python3.12/site-packages/tiktoken/model.py b/lib/python3.12/site-packages/tiktoken/model.py new file mode 100644 index 0000000000000000000000000000000000000000..5c669af4293dd09f415d5b9386fe58ddb3e8a212 --- /dev/null +++ b/lib/python3.12/site-packages/tiktoken/model.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from .core import Encoding +from .registry import get_encoding + +# TODO: these will likely be replaced by an API endpoint +MODEL_PREFIX_TO_ENCODING: dict[str, str] = { + "o1-": "o200k_base", + "o3-": "o200k_base", + "o4-mini-": "o200k_base", + # chat + "gpt-5-": "o200k_base", + "gpt-4.5-": "o200k_base", + "gpt-4.1-": "o200k_base", + "chatgpt-4o-": "o200k_base", + "gpt-4o-": "o200k_base", # e.g., gpt-4o-2024-05-13 + "gpt-4-": "cl100k_base", # e.g., gpt-4-0314, etc., plus gpt-4-32k + "gpt-3.5-turbo-": "cl100k_base", # e.g, gpt-3.5-turbo-0301, -0401, etc. + "gpt-35-turbo-": "cl100k_base", # Azure deployment name + "gpt-oss-": "o200k_harmony", + # fine-tuned + "ft:gpt-4o": "o200k_base", + "ft:gpt-4": "cl100k_base", + "ft:gpt-3.5-turbo": "cl100k_base", + "ft:davinci-002": "cl100k_base", + "ft:babbage-002": "cl100k_base", +} + +MODEL_TO_ENCODING: dict[str, str] = { + # reasoning + "o1": "o200k_base", + "o3": "o200k_base", + "o4-mini": "o200k_base", + # chat + "gpt-5": "o200k_base", + "gpt-4.1": "o200k_base", + "gpt-4o": "o200k_base", + "gpt-4": "cl100k_base", + "gpt-3.5-turbo": "cl100k_base", + "gpt-3.5": "cl100k_base", # Common shorthand + "gpt-35-turbo": "cl100k_base", # Azure deployment name + # base + "davinci-002": "cl100k_base", + "babbage-002": "cl100k_base", + # embeddings + "text-embedding-ada-002": "cl100k_base", + "text-embedding-3-small": "cl100k_base", + "text-embedding-3-large": "cl100k_base", + # DEPRECATED MODELS + # text (DEPRECATED) + "text-davinci-003": "p50k_base", + "text-davinci-002": "p50k_base", + "text-davinci-001": "r50k_base", + "text-curie-001": "r50k_base", + "text-babbage-001": "r50k_base", + "text-ada-001": "r50k_base", + "davinci": "r50k_base", + "curie": "r50k_base", + "babbage": "r50k_base", + "ada": "r50k_base", + # code (DEPRECATED) + "code-davinci-002": "p50k_base", + "code-davinci-001": "p50k_base", + "code-cushman-002": "p50k_base", + "code-cushman-001": "p50k_base", + "davinci-codex": "p50k_base", + "cushman-codex": "p50k_base", + # edit (DEPRECATED) + "text-davinci-edit-001": "p50k_edit", + "code-davinci-edit-001": "p50k_edit", + # old embeddings (DEPRECATED) + "text-similarity-davinci-001": "r50k_base", + "text-similarity-curie-001": "r50k_base", + "text-similarity-babbage-001": "r50k_base", + "text-similarity-ada-001": "r50k_base", + "text-search-davinci-doc-001": "r50k_base", + "text-search-curie-doc-001": "r50k_base", + "text-search-babbage-doc-001": "r50k_base", + "text-search-ada-doc-001": "r50k_base", + "code-search-babbage-code-001": "r50k_base", + "code-search-ada-code-001": "r50k_base", + # open source + "gpt2": "gpt2", + "gpt-2": "gpt2", # Maintains consistency with gpt-4 +} + + +def encoding_name_for_model(model_name: str) -> str: + """Returns the name of the encoding used by a model. + + Raises a KeyError if the model name is not recognised. + """ + encoding_name = None + if model_name in MODEL_TO_ENCODING: + encoding_name = MODEL_TO_ENCODING[model_name] + else: + # Check if the model matches a known prefix + # Prefix matching avoids needing library updates for every model version release + # Note that this can match on non-existent models (e.g., gpt-3.5-turbo-FAKE) + for model_prefix, model_encoding_name in MODEL_PREFIX_TO_ENCODING.items(): + if model_name.startswith(model_prefix): + return model_encoding_name + + if encoding_name is None: + raise KeyError( + f"Could not automatically map {model_name} to a tokeniser. " + "Please use `tiktoken.get_encoding` to explicitly get the tokeniser you expect." + ) from None + + return encoding_name + + +def encoding_for_model(model_name: str) -> Encoding: + """Returns the encoding used by a model. + + Raises a KeyError if the model name is not recognised. + """ + return get_encoding(encoding_name_for_model(model_name)) diff --git a/lib/python3.12/site-packages/tiktoken/py.typed b/lib/python3.12/site-packages/tiktoken/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/tiktoken/registry.py b/lib/python3.12/site-packages/tiktoken/registry.py new file mode 100644 index 0000000000000000000000000000000000000000..17c4574f6063e922f83f9fb000c5e1784ac299fe --- /dev/null +++ b/lib/python3.12/site-packages/tiktoken/registry.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import functools +import importlib +import pkgutil +import threading +from typing import Any, Callable, Sequence + +import tiktoken_ext + +import tiktoken +from tiktoken.core import Encoding + +_lock = threading.RLock() +ENCODINGS: dict[str, Encoding] = {} +ENCODING_CONSTRUCTORS: dict[str, Callable[[], dict[str, Any]]] | None = None + + +@functools.lru_cache +def _available_plugin_modules() -> Sequence[str]: + # tiktoken_ext is a namespace package + # submodules inside tiktoken_ext will be inspected for ENCODING_CONSTRUCTORS attributes + # - we use namespace package pattern so `pkgutil.iter_modules` is fast + # - it's a separate top-level package because namespace subpackages of non-namespace + # packages don't quite do what you want with editable installs + mods = [] + plugin_mods = pkgutil.iter_modules(tiktoken_ext.__path__, tiktoken_ext.__name__ + ".") + for _, mod_name, _ in plugin_mods: + mods.append(mod_name) + return mods + + +def _find_constructors() -> None: + global ENCODING_CONSTRUCTORS + with _lock: + if ENCODING_CONSTRUCTORS is not None: + return + ENCODING_CONSTRUCTORS = {} + + try: + for mod_name in _available_plugin_modules(): + mod = importlib.import_module(mod_name) + try: + constructors = mod.ENCODING_CONSTRUCTORS + except AttributeError as e: + raise ValueError( + f"tiktoken plugin {mod_name} does not define ENCODING_CONSTRUCTORS" + ) from e + for enc_name, constructor in constructors.items(): + if enc_name in ENCODING_CONSTRUCTORS: + raise ValueError( + f"Duplicate encoding name {enc_name} in tiktoken plugin {mod_name}" + ) + ENCODING_CONSTRUCTORS[enc_name] = constructor + except Exception: + # Ensure we idempotently raise errors + ENCODING_CONSTRUCTORS = None + raise + + + + +def get_encoding(encoding_name: str) -> Encoding: + if not isinstance(encoding_name, str): + raise ValueError(f"Expected a string in get_encoding, got {type(encoding_name)}") + + if encoding_name in ENCODINGS: + return ENCODINGS[encoding_name] + + with _lock: + if encoding_name in ENCODINGS: + return ENCODINGS[encoding_name] + + if ENCODING_CONSTRUCTORS is None: + _find_constructors() + assert ENCODING_CONSTRUCTORS is not None + + if encoding_name not in ENCODING_CONSTRUCTORS: + raise ValueError( + f"Unknown encoding {encoding_name}.\n" + f"Plugins found: {_available_plugin_modules()}\n" + f"tiktoken version: {tiktoken.__version__} (are you on latest?)" + ) + + constructor = ENCODING_CONSTRUCTORS[encoding_name] + enc = Encoding(**constructor()) + ENCODINGS[encoding_name] = enc + return enc + + +def list_encoding_names() -> list[str]: + with _lock: + if ENCODING_CONSTRUCTORS is None: + _find_constructors() + assert ENCODING_CONSTRUCTORS is not None + return list(ENCODING_CONSTRUCTORS) diff --git a/lib/python3.12/site-packages/torchvision/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..49207af20a4e3787ce6bed6c25efe0cd379f21cf Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/__pycache__/_internally_replaced_utils.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/__pycache__/_internally_replaced_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4700cd9d59ce84f04dfe0d4e27ceab75c34d7bb2 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/__pycache__/_internally_replaced_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/__pycache__/_meta_registrations.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/__pycache__/_meta_registrations.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..9095da0f760792c5aaf4fc03d8c90ff1f11cba2f Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/__pycache__/_meta_registrations.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/__pycache__/_utils.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/__pycache__/_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f1a2f077ba7bc865e35594a926cd6f72a02244e Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/__pycache__/_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/__pycache__/extension.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/__pycache__/extension.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..658d97c46a6ef34ee1a3ca4a690fbee03c7829a3 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/__pycache__/extension.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/__pycache__/utils.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..323ab9c2562eec00b2f33dffb5b646f340205fc7 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/__pycache__/utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/__pycache__/version.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/__pycache__/version.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..e25ab73f1ef8892ef2b879d03260ded0d93756cb Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/__pycache__/version.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/io/__init__.py b/lib/python3.12/site-packages/torchvision/io/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..03bd5d23cb2cf8e3acb67b7567e3ad9ef8061874 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/io/__init__.py @@ -0,0 +1,73 @@ +try: + from ._load_gpu_decoder import _HAS_GPU_VIDEO_DECODER +except ModuleNotFoundError: + _HAS_GPU_VIDEO_DECODER = False + +from ._video_opt import ( + _HAS_CPU_VIDEO_DECODER, + _HAS_VIDEO_OPT, + _probe_video_from_file, + _probe_video_from_memory, + _read_video_from_file, + _read_video_from_memory, + _read_video_timestamps_from_file, + _read_video_timestamps_from_memory, + Timebase, + VideoMetaData, +) +from .image import ( + decode_avif, + decode_gif, + decode_heic, + decode_image, + decode_jpeg, + decode_png, + decode_webp, + encode_jpeg, + encode_png, + ImageReadMode, + read_file, + read_image, + write_file, + write_jpeg, + write_png, +) +from .video import read_video, read_video_timestamps, write_video +from .video_reader import VideoReader + + +__all__ = [ + "write_video", + "read_video", + "read_video_timestamps", + "_read_video_from_file", + "_read_video_timestamps_from_file", + "_probe_video_from_file", + "_read_video_from_memory", + "_read_video_timestamps_from_memory", + "_probe_video_from_memory", + "_HAS_CPU_VIDEO_DECODER", + "_HAS_VIDEO_OPT", + "_HAS_GPU_VIDEO_DECODER", + "_read_video_clip_from_memory", + "_read_video_meta_data", + "VideoMetaData", + "Timebase", + "ImageReadMode", + "decode_image", + "decode_jpeg", + "decode_png", + "decode_avif", + "decode_heic", + "decode_webp", + "decode_gif", + "encode_jpeg", + "encode_png", + "read_file", + "read_image", + "write_file", + "write_jpeg", + "write_png", + "Video", + "VideoReader", +] diff --git a/lib/python3.12/site-packages/torchvision/io/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/io/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bb74ebbbc55ece995d04fa62745b2494ecf14d38 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/io/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/io/__pycache__/_load_gpu_decoder.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/io/__pycache__/_load_gpu_decoder.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..4cc15bc0db6d22bab00bac030913325ec1b1ec8b Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/io/__pycache__/_load_gpu_decoder.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/io/__pycache__/_video_opt.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/io/__pycache__/_video_opt.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..142f033c63f2b1132cf11f72f1bc8f41fd978319 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/io/__pycache__/_video_opt.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/io/__pycache__/image.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/io/__pycache__/image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..380afa9df94b6e71cc041cdffebea4b1bd0f7d85 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/io/__pycache__/image.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/io/__pycache__/video.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/io/__pycache__/video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cee0bc0b6d2bc95e10230bb4fef85c60d9658ed2 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/io/__pycache__/video.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/io/__pycache__/video_reader.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/io/__pycache__/video_reader.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..aa8a3b1cf9889b3842b469fe324e6f3ec8d2e6d3 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/io/__pycache__/video_reader.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/io/_load_gpu_decoder.py b/lib/python3.12/site-packages/torchvision/io/_load_gpu_decoder.py new file mode 100644 index 0000000000000000000000000000000000000000..cfd40c545d8201b67290e27bf74ce115774dace1 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/io/_load_gpu_decoder.py @@ -0,0 +1,8 @@ +from ..extension import _load_library + + +try: + _load_library("gpu_decoder") + _HAS_GPU_VIDEO_DECODER = True +except (ImportError, OSError): + _HAS_GPU_VIDEO_DECODER = False diff --git a/lib/python3.12/site-packages/torchvision/io/_video_opt.py b/lib/python3.12/site-packages/torchvision/io/_video_opt.py new file mode 100644 index 0000000000000000000000000000000000000000..69af045e7731882c949f9181f2f7e6d507edc4a1 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/io/_video_opt.py @@ -0,0 +1,513 @@ +import math +import warnings +from fractions import Fraction +from typing import Dict, List, Optional, Tuple, Union + +import torch + +from ..extension import _load_library + + +try: + _load_library("video_reader") + _HAS_CPU_VIDEO_DECODER = True +except (ImportError, OSError): + _HAS_CPU_VIDEO_DECODER = False + +_HAS_VIDEO_OPT = _HAS_CPU_VIDEO_DECODER # For BC +default_timebase = Fraction(0, 1) + + +# simple class for torch scripting +# the complex Fraction class from fractions module is not scriptable +class Timebase: + __annotations__ = {"numerator": int, "denominator": int} + __slots__ = ["numerator", "denominator"] + + def __init__( + self, + numerator: int, + denominator: int, + ) -> None: + self.numerator = numerator + self.denominator = denominator + + +class VideoMetaData: + __annotations__ = { + "has_video": bool, + "video_timebase": Timebase, + "video_duration": float, + "video_fps": float, + "has_audio": bool, + "audio_timebase": Timebase, + "audio_duration": float, + "audio_sample_rate": float, + } + __slots__ = [ + "has_video", + "video_timebase", + "video_duration", + "video_fps", + "has_audio", + "audio_timebase", + "audio_duration", + "audio_sample_rate", + ] + + def __init__(self) -> None: + self.has_video = False + self.video_timebase = Timebase(0, 1) + self.video_duration = 0.0 + self.video_fps = 0.0 + self.has_audio = False + self.audio_timebase = Timebase(0, 1) + self.audio_duration = 0.0 + self.audio_sample_rate = 0.0 + + +def _validate_pts(pts_range: Tuple[int, int]) -> None: + + if pts_range[0] > pts_range[1] > 0: + raise ValueError( + f"Start pts should not be smaller than end pts, got start pts: {pts_range[0]} and end pts: {pts_range[1]}" + ) + + +def _fill_info( + vtimebase: torch.Tensor, + vfps: torch.Tensor, + vduration: torch.Tensor, + atimebase: torch.Tensor, + asample_rate: torch.Tensor, + aduration: torch.Tensor, +) -> VideoMetaData: + """ + Build update VideoMetaData struct with info about the video + """ + meta = VideoMetaData() + if vtimebase.numel() > 0: + meta.video_timebase = Timebase(int(vtimebase[0].item()), int(vtimebase[1].item())) + timebase = vtimebase[0].item() / float(vtimebase[1].item()) + if vduration.numel() > 0: + meta.has_video = True + meta.video_duration = float(vduration.item()) * timebase + if vfps.numel() > 0: + meta.video_fps = float(vfps.item()) + if atimebase.numel() > 0: + meta.audio_timebase = Timebase(int(atimebase[0].item()), int(atimebase[1].item())) + timebase = atimebase[0].item() / float(atimebase[1].item()) + if aduration.numel() > 0: + meta.has_audio = True + meta.audio_duration = float(aduration.item()) * timebase + if asample_rate.numel() > 0: + meta.audio_sample_rate = float(asample_rate.item()) + + return meta + + +def _align_audio_frames( + aframes: torch.Tensor, aframe_pts: torch.Tensor, audio_pts_range: Tuple[int, int] +) -> torch.Tensor: + start, end = aframe_pts[0], aframe_pts[-1] + num_samples = aframes.size(0) + step_per_aframe = float(end - start + 1) / float(num_samples) + s_idx = 0 + e_idx = num_samples + if start < audio_pts_range[0]: + s_idx = int((audio_pts_range[0] - start) / step_per_aframe) + if audio_pts_range[1] != -1 and end > audio_pts_range[1]: + e_idx = int((audio_pts_range[1] - end) / step_per_aframe) + return aframes[s_idx:e_idx, :] + + +def _read_video_from_file( + filename: str, + seek_frame_margin: float = 0.25, + read_video_stream: bool = True, + video_width: int = 0, + video_height: int = 0, + video_min_dimension: int = 0, + video_max_dimension: int = 0, + video_pts_range: Tuple[int, int] = (0, -1), + video_timebase: Fraction = default_timebase, + read_audio_stream: bool = True, + audio_samples: int = 0, + audio_channels: int = 0, + audio_pts_range: Tuple[int, int] = (0, -1), + audio_timebase: Fraction = default_timebase, +) -> Tuple[torch.Tensor, torch.Tensor, VideoMetaData]: + """ + Reads a video from a file, returning both the video frames and the audio frames + + Args: + filename (str): path to the video file + seek_frame_margin (double, optional): seeking frame in the stream is imprecise. Thus, + when video_start_pts is specified, we seek the pts earlier by seek_frame_margin seconds + read_video_stream (int, optional): whether read video stream. If yes, set to 1. Otherwise, 0 + video_width/video_height/video_min_dimension/video_max_dimension (int): together decide + the size of decoded frames: + + - When video_width = 0, video_height = 0, video_min_dimension = 0, + and video_max_dimension = 0, keep the original frame resolution + - When video_width = 0, video_height = 0, video_min_dimension != 0, + and video_max_dimension = 0, keep the aspect ratio and resize the + frame so that shorter edge size is video_min_dimension + - When video_width = 0, video_height = 0, video_min_dimension = 0, + and video_max_dimension != 0, keep the aspect ratio and resize + the frame so that longer edge size is video_max_dimension + - When video_width = 0, video_height = 0, video_min_dimension != 0, + and video_max_dimension != 0, resize the frame so that shorter + edge size is video_min_dimension, and longer edge size is + video_max_dimension. The aspect ratio may not be preserved + - When video_width = 0, video_height != 0, video_min_dimension = 0, + and video_max_dimension = 0, keep the aspect ratio and resize + the frame so that frame video_height is $video_height + - When video_width != 0, video_height == 0, video_min_dimension = 0, + and video_max_dimension = 0, keep the aspect ratio and resize + the frame so that frame video_width is $video_width + - When video_width != 0, video_height != 0, video_min_dimension = 0, + and video_max_dimension = 0, resize the frame so that frame + video_width and video_height are set to $video_width and + $video_height, respectively + video_pts_range (list(int), optional): the start and end presentation timestamp of video stream + video_timebase (Fraction, optional): a Fraction rational number which denotes timebase in video stream + read_audio_stream (int, optional): whether read audio stream. If yes, set to 1. Otherwise, 0 + audio_samples (int, optional): audio sampling rate + audio_channels (int optional): audio channels + audio_pts_range (list(int), optional): the start and end presentation timestamp of audio stream + audio_timebase (Fraction, optional): a Fraction rational number which denotes time base in audio stream + + Returns + vframes (Tensor[T, H, W, C]): the `T` video frames + aframes (Tensor[L, K]): the audio frames, where `L` is the number of points and + `K` is the number of audio_channels + info (Dict): metadata for the video and audio. Can contain the fields video_fps (float) + and audio_fps (int) + """ + _validate_pts(video_pts_range) + _validate_pts(audio_pts_range) + + result = torch.ops.video_reader.read_video_from_file( + filename, + seek_frame_margin, + 0, # getPtsOnly + read_video_stream, + video_width, + video_height, + video_min_dimension, + video_max_dimension, + video_pts_range[0], + video_pts_range[1], + video_timebase.numerator, + video_timebase.denominator, + read_audio_stream, + audio_samples, + audio_channels, + audio_pts_range[0], + audio_pts_range[1], + audio_timebase.numerator, + audio_timebase.denominator, + ) + vframes, _vframe_pts, vtimebase, vfps, vduration, aframes, aframe_pts, atimebase, asample_rate, aduration = result + info = _fill_info(vtimebase, vfps, vduration, atimebase, asample_rate, aduration) + if aframes.numel() > 0: + # when audio stream is found + aframes = _align_audio_frames(aframes, aframe_pts, audio_pts_range) + return vframes, aframes, info + + +def _read_video_timestamps_from_file(filename: str) -> Tuple[List[int], List[int], VideoMetaData]: + """ + Decode all video- and audio frames in the video. Only pts + (presentation timestamp) is returned. The actual frame pixel data is not + copied. Thus, it is much faster than read_video(...) + """ + result = torch.ops.video_reader.read_video_from_file( + filename, + 0, # seek_frame_margin + 1, # getPtsOnly + 1, # read_video_stream + 0, # video_width + 0, # video_height + 0, # video_min_dimension + 0, # video_max_dimension + 0, # video_start_pts + -1, # video_end_pts + 0, # video_timebase_num + 1, # video_timebase_den + 1, # read_audio_stream + 0, # audio_samples + 0, # audio_channels + 0, # audio_start_pts + -1, # audio_end_pts + 0, # audio_timebase_num + 1, # audio_timebase_den + ) + _vframes, vframe_pts, vtimebase, vfps, vduration, _aframes, aframe_pts, atimebase, asample_rate, aduration = result + info = _fill_info(vtimebase, vfps, vduration, atimebase, asample_rate, aduration) + + vframe_pts = vframe_pts.numpy().tolist() + aframe_pts = aframe_pts.numpy().tolist() + return vframe_pts, aframe_pts, info + + +def _probe_video_from_file(filename: str) -> VideoMetaData: + """ + Probe a video file and return VideoMetaData with info about the video + """ + result = torch.ops.video_reader.probe_video_from_file(filename) + vtimebase, vfps, vduration, atimebase, asample_rate, aduration = result + info = _fill_info(vtimebase, vfps, vduration, atimebase, asample_rate, aduration) + return info + + +def _read_video_from_memory( + video_data: torch.Tensor, + seek_frame_margin: float = 0.25, + read_video_stream: int = 1, + video_width: int = 0, + video_height: int = 0, + video_min_dimension: int = 0, + video_max_dimension: int = 0, + video_pts_range: Tuple[int, int] = (0, -1), + video_timebase_numerator: int = 0, + video_timebase_denominator: int = 1, + read_audio_stream: int = 1, + audio_samples: int = 0, + audio_channels: int = 0, + audio_pts_range: Tuple[int, int] = (0, -1), + audio_timebase_numerator: int = 0, + audio_timebase_denominator: int = 1, +) -> Tuple[torch.Tensor, torch.Tensor]: + """ + Reads a video from memory, returning both the video frames as the audio frames + This function is torchscriptable. + + Args: + video_data (data type could be 1) torch.Tensor, dtype=torch.int8 or 2) python bytes): + compressed video content stored in either 1) torch.Tensor 2) python bytes + seek_frame_margin (double, optional): seeking frame in the stream is imprecise. + Thus, when video_start_pts is specified, we seek the pts earlier by seek_frame_margin seconds + read_video_stream (int, optional): whether read video stream. If yes, set to 1. Otherwise, 0 + video_width/video_height/video_min_dimension/video_max_dimension (int): together decide + the size of decoded frames: + + - When video_width = 0, video_height = 0, video_min_dimension = 0, + and video_max_dimension = 0, keep the original frame resolution + - When video_width = 0, video_height = 0, video_min_dimension != 0, + and video_max_dimension = 0, keep the aspect ratio and resize the + frame so that shorter edge size is video_min_dimension + - When video_width = 0, video_height = 0, video_min_dimension = 0, + and video_max_dimension != 0, keep the aspect ratio and resize + the frame so that longer edge size is video_max_dimension + - When video_width = 0, video_height = 0, video_min_dimension != 0, + and video_max_dimension != 0, resize the frame so that shorter + edge size is video_min_dimension, and longer edge size is + video_max_dimension. The aspect ratio may not be preserved + - When video_width = 0, video_height != 0, video_min_dimension = 0, + and video_max_dimension = 0, keep the aspect ratio and resize + the frame so that frame video_height is $video_height + - When video_width != 0, video_height == 0, video_min_dimension = 0, + and video_max_dimension = 0, keep the aspect ratio and resize + the frame so that frame video_width is $video_width + - When video_width != 0, video_height != 0, video_min_dimension = 0, + and video_max_dimension = 0, resize the frame so that frame + video_width and video_height are set to $video_width and + $video_height, respectively + video_pts_range (list(int), optional): the start and end presentation timestamp of video stream + video_timebase_numerator / video_timebase_denominator (float, optional): a rational + number which denotes timebase in video stream + read_audio_stream (int, optional): whether read audio stream. If yes, set to 1. Otherwise, 0 + audio_samples (int, optional): audio sampling rate + audio_channels (int optional): audio audio_channels + audio_pts_range (list(int), optional): the start and end presentation timestamp of audio stream + audio_timebase_numerator / audio_timebase_denominator (float, optional): + a rational number which denotes time base in audio stream + + Returns: + vframes (Tensor[T, H, W, C]): the `T` video frames + aframes (Tensor[L, K]): the audio frames, where `L` is the number of points and + `K` is the number of channels + """ + + _validate_pts(video_pts_range) + _validate_pts(audio_pts_range) + + if not isinstance(video_data, torch.Tensor): + with warnings.catch_warnings(): + # Ignore the warning because we actually don't modify the buffer in this function + warnings.filterwarnings("ignore", message="The given buffer is not writable") + video_data = torch.frombuffer(video_data, dtype=torch.uint8) + + result = torch.ops.video_reader.read_video_from_memory( + video_data, + seek_frame_margin, + 0, # getPtsOnly + read_video_stream, + video_width, + video_height, + video_min_dimension, + video_max_dimension, + video_pts_range[0], + video_pts_range[1], + video_timebase_numerator, + video_timebase_denominator, + read_audio_stream, + audio_samples, + audio_channels, + audio_pts_range[0], + audio_pts_range[1], + audio_timebase_numerator, + audio_timebase_denominator, + ) + + vframes, _vframe_pts, vtimebase, vfps, vduration, aframes, aframe_pts, atimebase, asample_rate, aduration = result + + if aframes.numel() > 0: + # when audio stream is found + aframes = _align_audio_frames(aframes, aframe_pts, audio_pts_range) + + return vframes, aframes + + +def _read_video_timestamps_from_memory( + video_data: torch.Tensor, +) -> Tuple[List[int], List[int], VideoMetaData]: + """ + Decode all frames in the video. Only pts (presentation timestamp) is returned. + The actual frame pixel data is not copied. Thus, read_video_timestamps(...) + is much faster than read_video(...) + """ + if not isinstance(video_data, torch.Tensor): + with warnings.catch_warnings(): + # Ignore the warning because we actually don't modify the buffer in this function + warnings.filterwarnings("ignore", message="The given buffer is not writable") + video_data = torch.frombuffer(video_data, dtype=torch.uint8) + result = torch.ops.video_reader.read_video_from_memory( + video_data, + 0, # seek_frame_margin + 1, # getPtsOnly + 1, # read_video_stream + 0, # video_width + 0, # video_height + 0, # video_min_dimension + 0, # video_max_dimension + 0, # video_start_pts + -1, # video_end_pts + 0, # video_timebase_num + 1, # video_timebase_den + 1, # read_audio_stream + 0, # audio_samples + 0, # audio_channels + 0, # audio_start_pts + -1, # audio_end_pts + 0, # audio_timebase_num + 1, # audio_timebase_den + ) + _vframes, vframe_pts, vtimebase, vfps, vduration, _aframes, aframe_pts, atimebase, asample_rate, aduration = result + info = _fill_info(vtimebase, vfps, vduration, atimebase, asample_rate, aduration) + + vframe_pts = vframe_pts.numpy().tolist() + aframe_pts = aframe_pts.numpy().tolist() + return vframe_pts, aframe_pts, info + + +def _probe_video_from_memory( + video_data: torch.Tensor, +) -> VideoMetaData: + """ + Probe a video in memory and return VideoMetaData with info about the video + This function is torchscriptable + """ + if not isinstance(video_data, torch.Tensor): + with warnings.catch_warnings(): + # Ignore the warning because we actually don't modify the buffer in this function + warnings.filterwarnings("ignore", message="The given buffer is not writable") + video_data = torch.frombuffer(video_data, dtype=torch.uint8) + result = torch.ops.video_reader.probe_video_from_memory(video_data) + vtimebase, vfps, vduration, atimebase, asample_rate, aduration = result + info = _fill_info(vtimebase, vfps, vduration, atimebase, asample_rate, aduration) + return info + + +def _read_video( + filename: str, + start_pts: Union[float, Fraction] = 0, + end_pts: Optional[Union[float, Fraction]] = None, + pts_unit: str = "pts", +) -> Tuple[torch.Tensor, torch.Tensor, Dict[str, float]]: + if end_pts is None: + end_pts = float("inf") + + if pts_unit == "pts": + warnings.warn( + "The pts_unit 'pts' gives wrong results and will be removed in a " + + "follow-up version. Please use pts_unit 'sec'." + ) + + info = _probe_video_from_file(filename) + + has_video = info.has_video + has_audio = info.has_audio + + def get_pts(time_base): + start_offset = start_pts + end_offset = end_pts + if pts_unit == "sec": + start_offset = int(math.floor(start_pts * (1 / time_base))) + if end_offset != float("inf"): + end_offset = int(math.ceil(end_pts * (1 / time_base))) + if end_offset == float("inf"): + end_offset = -1 + return start_offset, end_offset + + video_pts_range = (0, -1) + video_timebase = default_timebase + if has_video: + video_timebase = Fraction(info.video_timebase.numerator, info.video_timebase.denominator) + video_pts_range = get_pts(video_timebase) + + audio_pts_range = (0, -1) + audio_timebase = default_timebase + if has_audio: + audio_timebase = Fraction(info.audio_timebase.numerator, info.audio_timebase.denominator) + audio_pts_range = get_pts(audio_timebase) + + vframes, aframes, info = _read_video_from_file( + filename, + read_video_stream=True, + video_pts_range=video_pts_range, + video_timebase=video_timebase, + read_audio_stream=True, + audio_pts_range=audio_pts_range, + audio_timebase=audio_timebase, + ) + _info = {} + if has_video: + _info["video_fps"] = info.video_fps + if has_audio: + _info["audio_fps"] = info.audio_sample_rate + + return vframes, aframes, _info + + +def _read_video_timestamps( + filename: str, pts_unit: str = "pts" +) -> Tuple[Union[List[int], List[Fraction]], Optional[float]]: + if pts_unit == "pts": + warnings.warn( + "The pts_unit 'pts' gives wrong results and will be removed in a " + + "follow-up version. Please use pts_unit 'sec'." + ) + + pts: Union[List[int], List[Fraction]] + pts, _, info = _read_video_timestamps_from_file(filename) + + if pts_unit == "sec": + video_time_base = Fraction(info.video_timebase.numerator, info.video_timebase.denominator) + pts = [x * video_time_base for x in pts] + + video_fps = info.video_fps if info.has_video else None + + return pts, video_fps diff --git a/lib/python3.12/site-packages/torchvision/io/image.py b/lib/python3.12/site-packages/torchvision/io/image.py new file mode 100644 index 0000000000000000000000000000000000000000..023898f33c6d7b82a7161daee8dcf367e694df62 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/io/image.py @@ -0,0 +1,511 @@ +from enum import Enum +from typing import List, Union +from warnings import warn + +import torch + +from ..extension import _load_library +from ..utils import _log_api_usage_once + + +try: + _load_library("image") +except (ImportError, OSError) as e: + warn( + f"Failed to load image Python extension: '{e}'" + f"If you don't plan on using image functionality from `torchvision.io`, you can ignore this warning. " + f"Otherwise, there might be something wrong with your environment. " + f"Did you have `libjpeg` or `libpng` installed before building `torchvision` from source?" + ) + + +class ImageReadMode(Enum): + """Allow automatic conversion to RGB, RGBA, etc while decoding. + + .. note:: + + You don't need to use this struct, you can just pass strings to all + ``mode`` parameters, e.g. ``mode="RGB"``. + + The different available modes are the following. + + - UNCHANGED: loads the image as-is + - RGB: converts to RGB + - RGBA: converts to RGB with transparency (also aliased as RGB_ALPHA) + - GRAY: converts to grayscale + - GRAY_ALPHA: converts to grayscale with transparency + + .. note:: + + Some decoders won't support all possible values, e.g. GRAY and + GRAY_ALPHA are only supported for PNG and JPEG images. + """ + + UNCHANGED = 0 + GRAY = 1 + GRAY_ALPHA = 2 + RGB = 3 + RGB_ALPHA = 4 + RGBA = RGB_ALPHA # Alias for convenience + + +def read_file(path: str) -> torch.Tensor: + """ + Return the bytes contents of a file as a uint8 1D Tensor. + + Args: + path (str or ``pathlib.Path``): the path to the file to be read + + Returns: + data (Tensor) + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(read_file) + data = torch.ops.image.read_file(str(path)) + return data + + +def write_file(filename: str, data: torch.Tensor) -> None: + """ + Write the content of an uint8 1D tensor to a file. + + Args: + filename (str or ``pathlib.Path``): the path to the file to be written + data (Tensor): the contents to be written to the output file + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(write_file) + torch.ops.image.write_file(str(filename), data) + + +def decode_png( + input: torch.Tensor, + mode: ImageReadMode = ImageReadMode.UNCHANGED, + apply_exif_orientation: bool = False, +) -> torch.Tensor: + """ + Decodes a PNG image into a 3 dimensional RGB or grayscale Tensor. + + The values of the output tensor are in uint8 in [0, 255] for most cases. If + the image is a 16-bit png, then the output tensor is uint16 in [0, 65535] + (supported from torchvision ``0.21``). Since uint16 support is limited in + pytorch, we recommend calling + :func:`torchvision.transforms.v2.functional.to_dtype()` with ``scale=True`` + after this function to convert the decoded image into a uint8 or float + tensor. + + Args: + input (Tensor[1]): a one dimensional uint8 tensor containing + the raw bytes of the PNG image. + mode (str or ImageReadMode): The mode to convert the image to, e.g. "RGB". + Default is "UNCHANGED". See :class:`~torchvision.io.ImageReadMode` + for available modes. + apply_exif_orientation (bool): apply EXIF orientation transformation to the output tensor. + Default: False. + + Returns: + output (Tensor[image_channels, image_height, image_width]) + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(decode_png) + if isinstance(mode, str): + mode = ImageReadMode[mode.upper()] + output = torch.ops.image.decode_png(input, mode.value, apply_exif_orientation) + return output + + +def encode_png(input: torch.Tensor, compression_level: int = 6) -> torch.Tensor: + """ + Takes an input tensor in CHW layout and returns a buffer with the contents + of its corresponding PNG file. + + Args: + input (Tensor[channels, image_height, image_width]): int8 image tensor of + ``c`` channels, where ``c`` must 3 or 1. + compression_level (int): Compression factor for the resulting file, it must be a number + between 0 and 9. Default: 6 + + Returns: + Tensor[1]: A one dimensional int8 tensor that contains the raw bytes of the + PNG file. + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(encode_png) + output = torch.ops.image.encode_png(input, compression_level) + return output + + +def write_png(input: torch.Tensor, filename: str, compression_level: int = 6): + """ + Takes an input tensor in CHW layout (or HW in the case of grayscale images) + and saves it in a PNG file. + + Args: + input (Tensor[channels, image_height, image_width]): int8 image tensor of + ``c`` channels, where ``c`` must be 1 or 3. + filename (str or ``pathlib.Path``): Path to save the image. + compression_level (int): Compression factor for the resulting file, it must be a number + between 0 and 9. Default: 6 + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(write_png) + output = encode_png(input, compression_level) + write_file(filename, output) + + +def decode_jpeg( + input: Union[torch.Tensor, List[torch.Tensor]], + mode: ImageReadMode = ImageReadMode.UNCHANGED, + device: Union[str, torch.device] = "cpu", + apply_exif_orientation: bool = False, +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Decode JPEG image(s) into 3D RGB or grayscale Tensor(s), on CPU or CUDA. + + The values of the output tensor are uint8 between 0 and 255. + + .. note:: + When using a CUDA device, passing a list of tensors is more efficient than repeated individual calls to ``decode_jpeg``. + When using CPU the performance is equivalent. + The CUDA version of this function has explicitly been designed with thread-safety in mind. + This function does not return partial results in case of an error. + + Args: + input (Tensor[1] or list[Tensor[1]]): a (list of) one dimensional uint8 tensor(s) containing + the raw bytes of the JPEG image. The tensor(s) must be on CPU, + regardless of the ``device`` parameter. + mode (str or ImageReadMode): The mode to convert the image to, e.g. "RGB". + Default is "UNCHANGED". See :class:`~torchvision.io.ImageReadMode` + for available modes. + device (str or torch.device): The device on which the decoded image will + be stored. If a cuda device is specified, the image will be decoded + with `nvjpeg `_. This is only + supported for CUDA version >= 10.1 + + .. betastatus:: device parameter + + .. warning:: + There is a memory leak in the nvjpeg library for CUDA versions < 11.6. + Make sure to rely on CUDA 11.6 or above before using ``device="cuda"``. + apply_exif_orientation (bool): apply EXIF orientation transformation to the output tensor. + Default: False. Only implemented for JPEG format on CPU. + + Returns: + output (Tensor[image_channels, image_height, image_width] or list[Tensor[image_channels, image_height, image_width]]): + The values of the output tensor(s) are uint8 between 0 and 255. + ``output.device`` will be set to the specified ``device`` + + + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(decode_jpeg) + if isinstance(device, str): + device = torch.device(device) + if isinstance(mode, str): + mode = ImageReadMode[mode.upper()] + + if isinstance(input, list): + if len(input) == 0: + raise ValueError("Input list must contain at least one element") + if not all(isinstance(t, torch.Tensor) for t in input): + raise ValueError("All elements of the input list must be tensors.") + if not all(t.device.type == "cpu" for t in input): + raise ValueError("Input list must contain tensors on CPU.") + if device.type == "cuda": + return torch.ops.image.decode_jpegs_cuda(input, mode.value, device) + else: + return [torch.ops.image.decode_jpeg(img, mode.value, apply_exif_orientation) for img in input] + + else: # input is tensor + if input.device.type != "cpu": + raise ValueError("Input tensor must be a CPU tensor") + if device.type == "cuda": + return torch.ops.image.decode_jpegs_cuda([input], mode.value, device)[0] + else: + return torch.ops.image.decode_jpeg(input, mode.value, apply_exif_orientation) + + +def encode_jpeg( + input: Union[torch.Tensor, List[torch.Tensor]], quality: int = 75 +) -> Union[torch.Tensor, List[torch.Tensor]]: + """Encode RGB tensor(s) into raw encoded jpeg bytes, on CPU or CUDA. + + .. note:: + Passing a list of CUDA tensors is more efficient than repeated individual calls to ``encode_jpeg``. + For CPU tensors the performance is equivalent. + + Args: + input (Tensor[channels, image_height, image_width] or List[Tensor[channels, image_height, image_width]]): + (list of) uint8 image tensor(s) of ``c`` channels, where ``c`` must be 1 or 3 + quality (int): Quality of the resulting JPEG file(s). Must be a number between + 1 and 100. Default: 75 + + Returns: + output (Tensor[1] or list[Tensor[1]]): A (list of) one dimensional uint8 tensor(s) that contain the raw bytes of the JPEG file. + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(encode_jpeg) + if quality < 1 or quality > 100: + raise ValueError("Image quality should be a positive number between 1 and 100") + if isinstance(input, list): + if not input: + raise ValueError("encode_jpeg requires at least one input tensor when a list is passed") + if input[0].device.type == "cuda": + return torch.ops.image.encode_jpegs_cuda(input, quality) + else: + return [torch.ops.image.encode_jpeg(image, quality) for image in input] + else: # single input tensor + if input.device.type == "cuda": + return torch.ops.image.encode_jpegs_cuda([input], quality)[0] + else: + return torch.ops.image.encode_jpeg(input, quality) + + +def write_jpeg(input: torch.Tensor, filename: str, quality: int = 75): + """ + Takes an input tensor in CHW layout and saves it in a JPEG file. + + Args: + input (Tensor[channels, image_height, image_width]): int8 image tensor of ``c`` + channels, where ``c`` must be 1 or 3. + filename (str or ``pathlib.Path``): Path to save the image. + quality (int): Quality of the resulting JPEG file, it must be a number + between 1 and 100. Default: 75 + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(write_jpeg) + output = encode_jpeg(input, quality) + assert isinstance(output, torch.Tensor) # Needed for torchscript + write_file(filename, output) + + +def decode_image( + input: Union[torch.Tensor, str], + mode: ImageReadMode = ImageReadMode.UNCHANGED, + apply_exif_orientation: bool = False, +) -> torch.Tensor: + """Decode an image into a uint8 tensor, from a path or from raw encoded bytes. + + Currently supported image formats are jpeg, png, gif and webp. + + The values of the output tensor are in uint8 in [0, 255] for most cases. + + If the image is a 16-bit png, then the output tensor is uint16 in [0, 65535] + (supported from torchvision ``0.21``). Since uint16 support is limited in + pytorch, we recommend calling + :func:`torchvision.transforms.v2.functional.to_dtype()` with ``scale=True`` + after this function to convert the decoded image into a uint8 or float + tensor. + + .. note:: + + ``decode_image()`` doesn't work yet on AVIF or HEIC images. For these + formats, directly call :func:`~torchvision.io.decode_avif` or + :func:`~torchvision.io.decode_heic`. + + Args: + input (Tensor or str or ``pathlib.Path``): The image to decode. If a + tensor is passed, it must be one dimensional uint8 tensor containing + the raw bytes of the image. Otherwise, this must be a path to the image file. + mode (str or ImageReadMode): The mode to convert the image to, e.g. "RGB". + Default is "UNCHANGED". See :class:`~torchvision.io.ImageReadMode` + for available modes. + apply_exif_orientation (bool): apply EXIF orientation transformation to the output tensor. + Only applies to JPEG and PNG images. Default: False. + + Returns: + output (Tensor[image_channels, image_height, image_width]) + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(decode_image) + if not isinstance(input, torch.Tensor): + input = read_file(str(input)) + if isinstance(mode, str): + mode = ImageReadMode[mode.upper()] + output = torch.ops.image.decode_image(input, mode.value, apply_exif_orientation) + return output + + +def read_image( + path: str, + mode: ImageReadMode = ImageReadMode.UNCHANGED, + apply_exif_orientation: bool = False, +) -> torch.Tensor: + """[OBSOLETE] Use :func:`~torchvision.io.decode_image` instead.""" + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(read_image) + data = read_file(path) + return decode_image(data, mode, apply_exif_orientation=apply_exif_orientation) + + +def decode_gif(input: torch.Tensor) -> torch.Tensor: + """ + Decode a GIF image into a 3 or 4 dimensional RGB Tensor. + + The values of the output tensor are uint8 between 0 and 255. + The output tensor has shape ``(C, H, W)`` if there is only one image in the + GIF, and ``(N, C, H, W)`` if there are ``N`` images. + + Args: + input (Tensor[1]): a one dimensional contiguous uint8 tensor containing + the raw bytes of the GIF image. + + Returns: + output (Tensor[image_channels, image_height, image_width] or Tensor[num_images, image_channels, image_height, image_width]) + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(decode_gif) + return torch.ops.image.decode_gif(input) + + +def decode_webp( + input: torch.Tensor, + mode: ImageReadMode = ImageReadMode.UNCHANGED, +) -> torch.Tensor: + """ + Decode a WEBP image into a 3 dimensional RGB[A] Tensor. + + The values of the output tensor are uint8 between 0 and 255. + + Args: + input (Tensor[1]): a one dimensional contiguous uint8 tensor containing + the raw bytes of the WEBP image. + mode (str or ImageReadMode): The mode to convert the image to, e.g. "RGB". + Default is "UNCHANGED". See :class:`~torchvision.io.ImageReadMode` + for available modes. + + Returns: + Decoded image (Tensor[image_channels, image_height, image_width]) + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(decode_webp) + if isinstance(mode, str): + mode = ImageReadMode[mode.upper()] + return torch.ops.image.decode_webp(input, mode.value) + + +# TODO_AVIF_HEIC: Better support for torchscript. Scripting decode_avif of +# decode_heic currently fails, mainly because of the logic +# _load_extra_decoders_once() (using global variables, try/except statements, +# etc.). +# The ops (torch.ops.extra_decoders_ns.decode_*) are otherwise torchscript-able, +# and users who need torchscript can always just wrap those. + +# TODO_AVIF_HEIC: decode_image() should work for those. The key technical issue +# we have here is that the format detection logic of decode_image() is +# implemented in torchvision, and torchvision has zero knowledge of +# torchvision-extra-decoders, so we cannot call the AVIF/HEIC C++ decoders +# (those in torchvision-extra-decoders) from there. +# A trivial check that could be done within torchvision would be to check the +# file extension, if a path was passed. We could also just implement the +# AVIF/HEIC detection logic in Python as a fallback, if the file detection +# didn't find any format. In any case: properly determining whether a file is +# HEIC is far from trivial, and relying on libmagic would probably be best + + +_EXTRA_DECODERS_ALREADY_LOADED = False + + +def _load_extra_decoders_once(): + global _EXTRA_DECODERS_ALREADY_LOADED + if _EXTRA_DECODERS_ALREADY_LOADED: + return + + try: + import torchvision_extra_decoders + + # torchvision-extra-decoders only supports linux for now. BUT, users on + # e.g. MacOS can still install it: they will get the pure-python + # 0.0.0.dev version: + # https://pypi.org/project/torchvision-extra-decoders/0.0.0.dev0, which + # is a dummy version that was created to reserve the namespace on PyPI. + # We have to check that expose_extra_decoders() exists for those users, + # so we can properly error on non-Linux archs. + assert hasattr(torchvision_extra_decoders, "expose_extra_decoders") + except (AssertionError, ImportError) as e: + raise RuntimeError( + "In order to enable the AVIF and HEIC decoding capabilities of " + "torchvision, you need to `pip install torchvision-extra-decoders`. " + "Just install the package, you don't need to update your code. " + "This is only supported on Linux, and this feature is still in BETA stage. " + "Please let us know of any issue: https://github.com/pytorch/vision/issues/new/choose. " + "Note that `torchvision-extra-decoders` is released under the LGPL license. " + ) from e + + # This will expose torch.ops.extra_decoders_ns.decode_avif and torch.ops.extra_decoders_ns.decode_heic + torchvision_extra_decoders.expose_extra_decoders() + + _EXTRA_DECODERS_ALREADY_LOADED = True + + +def decode_avif(input: torch.Tensor, mode: ImageReadMode = ImageReadMode.UNCHANGED) -> torch.Tensor: + """Decode an AVIF image into a 3 dimensional RGB[A] Tensor. + + .. warning:: + In order to enable the AVIF decoding capabilities of torchvision, you + first need to run ``pip install torchvision-extra-decoders``. Just + install the package, you don't need to update your code. This is only + supported on Linux, and this feature is still in BETA stage. Please let + us know of any issue: + https://github.com/pytorch/vision/issues/new/choose. Note that + `torchvision-extra-decoders + `_ is + released under the LGPL license. + + The values of the output tensor are in uint8 in [0, 255] for most images. If + the image has a bit-depth of more than 8, then the output tensor is uint16 + in [0, 65535]. Since uint16 support is limited in pytorch, we recommend + calling :func:`torchvision.transforms.v2.functional.to_dtype()` with + ``scale=True`` after this function to convert the decoded image into a uint8 + or float tensor. + + Args: + input (Tensor[1]): a one dimensional contiguous uint8 tensor containing + the raw bytes of the AVIF image. + mode (str or ImageReadMode): The mode to convert the image to, e.g. "RGB". + Default is "UNCHANGED". See :class:`~torchvision.io.ImageReadMode` + for available modes. + + Returns: + Decoded image (Tensor[image_channels, image_height, image_width]) + """ + _load_extra_decoders_once() + if input.dtype != torch.uint8: + raise RuntimeError(f"Input tensor must have uint8 data type, got {input.dtype}") + return torch.ops.extra_decoders_ns.decode_avif(input, mode.value) + + +def decode_heic(input: torch.Tensor, mode: ImageReadMode = ImageReadMode.UNCHANGED) -> torch.Tensor: + """Decode an HEIC image into a 3 dimensional RGB[A] Tensor. + + .. warning:: + In order to enable the AVIF decoding capabilities of torchvision, you + first need to run ``pip install torchvision-extra-decoders``. Just + install the package, you don't need to update your code. This is only + supported on Linux, and this feature is still in BETA stage. Please let + us know of any issue: + https://github.com/pytorch/vision/issues/new/choose. Note that + `torchvision-extra-decoders + `_ is + released under the LGPL license. + + The values of the output tensor are in uint8 in [0, 255] for most images. If + the image has a bit-depth of more than 8, then the output tensor is uint16 + in [0, 65535]. Since uint16 support is limited in pytorch, we recommend + calling :func:`torchvision.transforms.v2.functional.to_dtype()` with + ``scale=True`` after this function to convert the decoded image into a uint8 + or float tensor. + + Args: + input (Tensor[1]): a one dimensional contiguous uint8 tensor containing + the raw bytes of the HEIC image. + mode (str or ImageReadMode): The mode to convert the image to, e.g. "RGB". + Default is "UNCHANGED". See :class:`~torchvision.io.ImageReadMode` + for available modes. + + Returns: + Decoded image (Tensor[image_channels, image_height, image_width]) + """ + _load_extra_decoders_once() + if input.dtype != torch.uint8: + raise RuntimeError(f"Input tensor must have uint8 data type, got {input.dtype}") + return torch.ops.extra_decoders_ns.decode_heic(input, mode.value) diff --git a/lib/python3.12/site-packages/torchvision/io/video.py b/lib/python3.12/site-packages/torchvision/io/video.py new file mode 100644 index 0000000000000000000000000000000000000000..2e3dbed65a279f0746a0fd38a0ae98d870938653 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/io/video.py @@ -0,0 +1,466 @@ +import gc +import math +import os +import re +import warnings +from fractions import Fraction +from typing import Any, Dict, List, Optional, Tuple, Union + +import numpy as np +import torch + +from ..utils import _log_api_usage_once +from . import _video_opt + +try: + import av + + av.logging.set_level(av.logging.ERROR) + if not hasattr(av.video.frame.VideoFrame, "pict_type"): + av = ImportError( + """\ +Your version of PyAV is too old for the necessary video operations in torchvision. +If you are on Python 3.5, you will have to build from source (the conda-forge +packages are not up-to-date). See +https://github.com/mikeboers/PyAV#installation for instructions on how to +install PyAV on your system. +""" + ) + try: + FFmpegError = av.FFmpegError # from av 14 https://github.com/PyAV-Org/PyAV/blob/main/CHANGELOG.rst + except AttributeError: + FFmpegError = av.AVError +except ImportError: + av = ImportError( + """\ +PyAV is not installed, and is necessary for the video operations in torchvision. +See https://github.com/mikeboers/PyAV#installation for instructions on how to +install PyAV on your system. +""" + ) + + +def _check_av_available() -> None: + if isinstance(av, Exception): + raise av + + +def _av_available() -> bool: + return not isinstance(av, Exception) + + +# PyAV has some reference cycles +_CALLED_TIMES = 0 +_GC_COLLECTION_INTERVAL = 10 + + +def write_video( + filename: str, + video_array: torch.Tensor, + fps: float, + video_codec: str = "libx264", + options: Optional[Dict[str, Any]] = None, + audio_array: Optional[torch.Tensor] = None, + audio_fps: Optional[float] = None, + audio_codec: Optional[str] = None, + audio_options: Optional[Dict[str, Any]] = None, +) -> None: + """ + Writes a 4d tensor in [T, H, W, C] format in a video file. + + This function relies on PyAV (therefore, ultimately FFmpeg) to encode + videos, you can get more fine-grained control by referring to the other + options at your disposal within `the FFMpeg wiki + `_. + + .. warning:: + + In the near future, we intend to centralize PyTorch's video decoding + capabilities within the `torchcodec + `_ project. We encourage you to + try it out and share your feedback, as the torchvision video decoders + will eventually be deprecated. + + Args: + filename (str): path where the video will be saved + video_array (Tensor[T, H, W, C]): tensor containing the individual frames, + as a uint8 tensor in [T, H, W, C] format + fps (Number): video frames per second + video_codec (str): the name of the video codec, i.e. "libx264", "h264", etc. + options (Dict): dictionary containing options to be passed into the PyAV video stream. + The list of options is codec-dependent and can all + be found from `the FFMpeg wiki `_. + audio_array (Tensor[C, N]): tensor containing the audio, where C is the number of channels + and N is the number of samples + audio_fps (Number): audio sample rate, typically 44100 or 48000 + audio_codec (str): the name of the audio codec, i.e. "mp3", "aac", etc. + audio_options (Dict): dictionary containing options to be passed into the PyAV audio stream. + The list of options is codec-dependent and can all + be found from `the FFMpeg wiki `_. + + Examples:: + >>> # Creating libx264 video with CRF 17, for visually lossless footage: + >>> + >>> from torchvision.io import write_video + >>> # 1000 frames of 100x100, 3-channel image. + >>> vid = torch.randn(1000, 100, 100, 3, dtype = torch.uint8) + >>> write_video("video.mp4", options = {"crf": "17"}) + + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(write_video) + _check_av_available() + video_array = torch.as_tensor(video_array, dtype=torch.uint8).numpy(force=True) + + # PyAV does not support floating point numbers with decimal point + # and will throw OverflowException in case this is not the case + if isinstance(fps, float): + fps = np.round(fps) + + with av.open(filename, mode="w") as container: + stream = container.add_stream(video_codec, rate=fps) + stream.width = video_array.shape[2] + stream.height = video_array.shape[1] + stream.pix_fmt = "yuv420p" if video_codec != "libx264rgb" else "rgb24" + stream.options = options or {} + + if audio_array is not None: + audio_format_dtypes = { + "dbl": " 1 else "mono" + audio_sample_fmt = container.streams.audio[0].format.name + + format_dtype = np.dtype(audio_format_dtypes[audio_sample_fmt]) + audio_array = torch.as_tensor(audio_array).numpy(force=True).astype(format_dtype) + + frame = av.AudioFrame.from_ndarray(audio_array, format=audio_sample_fmt, layout=audio_layout) + + frame.sample_rate = audio_fps + + for packet in a_stream.encode(frame): + container.mux(packet) + + for packet in a_stream.encode(): + container.mux(packet) + + for img in video_array: + frame = av.VideoFrame.from_ndarray(img, format="rgb24") + try: + frame.pict_type = "NONE" + except TypeError: + from av.video.frame import PictureType # noqa + + frame.pict_type = PictureType.NONE + + for packet in stream.encode(frame): + container.mux(packet) + + # Flush stream + for packet in stream.encode(): + container.mux(packet) + + +def _read_from_stream( + container: "av.container.Container", + start_offset: float, + end_offset: float, + pts_unit: str, + stream: "av.stream.Stream", + stream_name: Dict[str, Optional[Union[int, Tuple[int, ...], List[int]]]], +) -> List["av.frame.Frame"]: + global _CALLED_TIMES, _GC_COLLECTION_INTERVAL + _CALLED_TIMES += 1 + if _CALLED_TIMES % _GC_COLLECTION_INTERVAL == _GC_COLLECTION_INTERVAL - 1: + gc.collect() + + if pts_unit == "sec": + # TODO: we should change all of this from ground up to simply take + # sec and convert to MS in C++ + start_offset = int(math.floor(start_offset * (1 / stream.time_base))) + if end_offset != float("inf"): + end_offset = int(math.ceil(end_offset * (1 / stream.time_base))) + else: + warnings.warn("The pts_unit 'pts' gives wrong results. Please use pts_unit 'sec'.") + + frames = {} + should_buffer = True + max_buffer_size = 5 + if stream.type == "video": + # DivX-style packed B-frames can have out-of-order pts (2 frames in a single pkt) + # so need to buffer some extra frames to sort everything + # properly + extradata = stream.codec_context.extradata + # overly complicated way of finding if `divx_packed` is set, following + # https://github.com/FFmpeg/FFmpeg/commit/d5a21172283572af587b3d939eba0091484d3263 + if extradata and b"DivX" in extradata: + # can't use regex directly because of some weird characters sometimes... + pos = extradata.find(b"DivX") + d = extradata[pos:] + o = re.search(rb"DivX(\d+)Build(\d+)(\w)", d) + if o is None: + o = re.search(rb"DivX(\d+)b(\d+)(\w)", d) + if o is not None: + should_buffer = o.group(3) == b"p" + seek_offset = start_offset + # some files don't seek to the right location, so better be safe here + seek_offset = max(seek_offset - 1, 0) + if should_buffer: + # FIXME this is kind of a hack, but we will jump to the previous keyframe + # so this will be safe + seek_offset = max(seek_offset - max_buffer_size, 0) + try: + # TODO check if stream needs to always be the video stream here or not + container.seek(seek_offset, any_frame=False, backward=True, stream=stream) + except FFmpegError: + # TODO add some warnings in this case + # print("Corrupted file?", container.name) + return [] + buffer_count = 0 + try: + for _idx, frame in enumerate(container.decode(**stream_name)): + frames[frame.pts] = frame + if frame.pts >= end_offset: + if should_buffer and buffer_count < max_buffer_size: + buffer_count += 1 + continue + break + except FFmpegError: + # TODO add a warning + pass + # ensure that the results are sorted wrt the pts + result = [frames[i] for i in sorted(frames) if start_offset <= frames[i].pts <= end_offset] + if len(frames) > 0 and start_offset > 0 and start_offset not in frames: + # if there is no frame that exactly matches the pts of start_offset + # add the last frame smaller than start_offset, to guarantee that + # we will have all the necessary data. This is most useful for audio + preceding_frames = [i for i in frames if i < start_offset] + if len(preceding_frames) > 0: + first_frame_pts = max(preceding_frames) + result.insert(0, frames[first_frame_pts]) + return result + + +def _align_audio_frames( + aframes: torch.Tensor, audio_frames: List["av.frame.Frame"], ref_start: int, ref_end: float +) -> torch.Tensor: + start, end = audio_frames[0].pts, audio_frames[-1].pts + total_aframes = aframes.shape[1] + step_per_aframe = (end - start + 1) / total_aframes + s_idx = 0 + e_idx = total_aframes + if start < ref_start: + s_idx = int((ref_start - start) / step_per_aframe) + if end > ref_end: + e_idx = int((ref_end - end) / step_per_aframe) + return aframes[:, s_idx:e_idx] + + +def read_video( + filename: str, + start_pts: Union[float, Fraction] = 0, + end_pts: Optional[Union[float, Fraction]] = None, + pts_unit: str = "pts", + output_format: str = "THWC", +) -> Tuple[torch.Tensor, torch.Tensor, Dict[str, Any]]: + """ + Reads a video from a file, returning both the video frames and the audio frames + + .. warning:: + + In the near future, we intend to centralize PyTorch's video decoding + capabilities within the `torchcodec + `_ project. We encourage you to + try it out and share your feedback, as the torchvision video decoders + will eventually be deprecated. + + Args: + filename (str): path to the video file. If using the pyav backend, this can be whatever ``av.open`` accepts. + start_pts (int if pts_unit = 'pts', float / Fraction if pts_unit = 'sec', optional): + The start presentation time of the video + end_pts (int if pts_unit = 'pts', float / Fraction if pts_unit = 'sec', optional): + The end presentation time + pts_unit (str, optional): unit in which start_pts and end_pts values will be interpreted, + either 'pts' or 'sec'. Defaults to 'pts'. + output_format (str, optional): The format of the output video tensors. Can be either "THWC" (default) or "TCHW". + + Returns: + vframes (Tensor[T, H, W, C] or Tensor[T, C, H, W]): the `T` video frames + aframes (Tensor[K, L]): the audio frames, where `K` is the number of channels and `L` is the number of points + info (Dict): metadata for the video and audio. Can contain the fields video_fps (float) and audio_fps (int) + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(read_video) + + output_format = output_format.upper() + if output_format not in ("THWC", "TCHW"): + raise ValueError(f"output_format should be either 'THWC' or 'TCHW', got {output_format}.") + + from torchvision import get_video_backend + + if get_video_backend() != "pyav": + if not os.path.exists(filename): + raise RuntimeError(f"File not found: {filename}") + vframes, aframes, info = _video_opt._read_video(filename, start_pts, end_pts, pts_unit) + else: + _check_av_available() + + if end_pts is None: + end_pts = float("inf") + + if end_pts < start_pts: + raise ValueError( + f"end_pts should be larger than start_pts, got start_pts={start_pts} and end_pts={end_pts}" + ) + + info = {} + video_frames = [] + audio_frames = [] + audio_timebase = _video_opt.default_timebase + + try: + with av.open(filename, metadata_errors="ignore") as container: + if container.streams.audio: + audio_timebase = container.streams.audio[0].time_base + if container.streams.video: + video_frames = _read_from_stream( + container, + start_pts, + end_pts, + pts_unit, + container.streams.video[0], + {"video": 0}, + ) + video_fps = container.streams.video[0].average_rate + # guard against potentially corrupted files + if video_fps is not None: + info["video_fps"] = float(video_fps) + + if container.streams.audio: + audio_frames = _read_from_stream( + container, + start_pts, + end_pts, + pts_unit, + container.streams.audio[0], + {"audio": 0}, + ) + info["audio_fps"] = container.streams.audio[0].rate + + except FFmpegError: + # TODO raise a warning? + pass + + vframes_list = [frame.to_rgb().to_ndarray() for frame in video_frames] + aframes_list = [frame.to_ndarray() for frame in audio_frames] + + if vframes_list: + vframes = torch.as_tensor(np.stack(vframes_list)) + else: + vframes = torch.empty((0, 1, 1, 3), dtype=torch.uint8) + + if aframes_list: + aframes = np.concatenate(aframes_list, 1) + aframes = torch.as_tensor(aframes) + if pts_unit == "sec": + start_pts = int(math.floor(start_pts * (1 / audio_timebase))) + if end_pts != float("inf"): + end_pts = int(math.ceil(end_pts * (1 / audio_timebase))) + aframes = _align_audio_frames(aframes, audio_frames, start_pts, end_pts) + else: + aframes = torch.empty((1, 0), dtype=torch.float32) + + if output_format == "TCHW": + # [T,H,W,C] --> [T,C,H,W] + vframes = vframes.permute(0, 3, 1, 2) + + return vframes, aframes, info + + +def _can_read_timestamps_from_packets(container: "av.container.Container") -> bool: + extradata = container.streams[0].codec_context.extradata + if extradata is None: + return False + if b"Lavc" in extradata: + return True + return False + + +def _decode_video_timestamps(container: "av.container.Container") -> List[int]: + if _can_read_timestamps_from_packets(container): + # fast path + return [x.pts for x in container.demux(video=0) if x.pts is not None] + else: + return [x.pts for x in container.decode(video=0) if x.pts is not None] + + +def read_video_timestamps(filename: str, pts_unit: str = "pts") -> Tuple[List[int], Optional[float]]: + """ + List the video frames timestamps. + + .. warning:: + + In the near future, we intend to centralize PyTorch's video decoding + capabilities within the `torchcodec + `_ project. We encourage you to + try it out and share your feedback, as the torchvision video decoders + will eventually be deprecated. + + Note that the function decodes the whole video frame-by-frame. + + Args: + filename (str): path to the video file + pts_unit (str, optional): unit in which timestamp values will be returned + either 'pts' or 'sec'. Defaults to 'pts'. + + Returns: + pts (List[int] if pts_unit = 'pts', List[Fraction] if pts_unit = 'sec'): + presentation timestamps for each one of the frames in the video. + video_fps (float, optional): the frame rate for the video + + """ + if not torch.jit.is_scripting() and not torch.jit.is_tracing(): + _log_api_usage_once(read_video_timestamps) + from torchvision import get_video_backend + + if get_video_backend() != "pyav": + return _video_opt._read_video_timestamps(filename, pts_unit) + + _check_av_available() + + video_fps = None + pts = [] + + try: + with av.open(filename, metadata_errors="ignore") as container: + if container.streams.video: + video_stream = container.streams.video[0] + video_time_base = video_stream.time_base + try: + pts = _decode_video_timestamps(container) + except FFmpegError: + warnings.warn(f"Failed decoding frames for file {filename}") + video_fps = float(video_stream.average_rate) + except FFmpegError as e: + msg = f"Failed to open container for {filename}; Caught error: {e}" + warnings.warn(msg, RuntimeWarning) + + pts.sort() + + if pts_unit == "sec": + pts = [x * video_time_base for x in pts] + + return pts, video_fps diff --git a/lib/python3.12/site-packages/torchvision/io/video_reader.py b/lib/python3.12/site-packages/torchvision/io/video_reader.py new file mode 100644 index 0000000000000000000000000000000000000000..cf319fe288e1e1a67cc94cb798ab8a9212671101 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/io/video_reader.py @@ -0,0 +1,294 @@ +import io +import warnings + +from typing import Any, Dict, Iterator + +import torch + +from ..utils import _log_api_usage_once + +from ._video_opt import _HAS_CPU_VIDEO_DECODER + +if _HAS_CPU_VIDEO_DECODER: + + def _has_video_opt() -> bool: + return True + +else: + + def _has_video_opt() -> bool: + return False + + +try: + import av + + av.logging.set_level(av.logging.ERROR) + if not hasattr(av.video.frame.VideoFrame, "pict_type"): + av = ImportError( + """\ +Your version of PyAV is too old for the necessary video operations in torchvision. +If you are on Python 3.5, you will have to build from source (the conda-forge +packages are not up-to-date). See +https://github.com/mikeboers/PyAV#installation for instructions on how to +install PyAV on your system. +""" + ) +except ImportError: + av = ImportError( + """\ +PyAV is not installed, and is necessary for the video operations in torchvision. +See https://github.com/mikeboers/PyAV#installation for instructions on how to +install PyAV on your system. +""" + ) + + +class VideoReader: + """ + Fine-grained video-reading API. + Supports frame-by-frame reading of various streams from a single video + container. Much like previous video_reader API it supports the following + backends: video_reader, pyav, and cuda. + Backends can be set via `torchvision.set_video_backend` function. + + .. warning:: + + In the near future, we intend to centralize PyTorch's video decoding + capabilities within the `torchcodec + `_ project. We encourage you to + try it out and share your feedback, as the torchvision video decoders + will eventually be deprecated. + + .. betastatus:: VideoReader class + + Example: + The following examples creates a :mod:`VideoReader` object, seeks into 2s + point, and returns a single frame:: + + import torchvision + video_path = "path_to_a_test_video" + reader = torchvision.io.VideoReader(video_path, "video") + reader.seek(2.0) + frame = next(reader) + + :mod:`VideoReader` implements the iterable API, which makes it suitable to + using it in conjunction with :mod:`itertools` for more advanced reading. + As such, we can use a :mod:`VideoReader` instance inside for loops:: + + reader.seek(2) + for frame in reader: + frames.append(frame['data']) + # additionally, `seek` implements a fluent API, so we can do + for frame in reader.seek(2): + frames.append(frame['data']) + + With :mod:`itertools`, we can read all frames between 2 and 5 seconds with the + following code:: + + for frame in itertools.takewhile(lambda x: x['pts'] <= 5, reader.seek(2)): + frames.append(frame['data']) + + and similarly, reading 10 frames after the 2s timestamp can be achieved + as follows:: + + for frame in itertools.islice(reader.seek(2), 10): + frames.append(frame['data']) + + .. note:: + + Each stream descriptor consists of two parts: stream type (e.g. 'video') and + a unique stream id (which are determined by the video encoding). + In this way, if the video container contains multiple + streams of the same type, users can access the one they want. + If only stream type is passed, the decoder auto-detects first stream of that type. + + Args: + src (string, bytes object, or tensor): The media source. + If string-type, it must be a file path supported by FFMPEG. + If bytes, should be an in-memory representation of a file supported by FFMPEG. + If Tensor, it is interpreted internally as byte buffer. + It must be one-dimensional, of type ``torch.uint8``. + + stream (string, optional): descriptor of the required stream, followed by the stream id, + in the format ``{stream_type}:{stream_id}``. Defaults to ``"video:0"``. + Currently available options include ``['video', 'audio']`` + + num_threads (int, optional): number of threads used by the codec to decode video. + Default value (0) enables multithreading with codec-dependent heuristic. The performance + will depend on the version of FFMPEG codecs supported. + """ + + def __init__( + self, + src: str, + stream: str = "video", + num_threads: int = 0, + ) -> None: + _log_api_usage_once(self) + from .. import get_video_backend + + self.backend = get_video_backend() + if isinstance(src, str): + if not src: + raise ValueError("src cannot be empty") + elif isinstance(src, bytes): + if self.backend in ["cuda"]: + raise RuntimeError( + "VideoReader cannot be initialized from bytes object when using cuda or pyav backend." + ) + elif self.backend == "pyav": + src = io.BytesIO(src) + else: + with warnings.catch_warnings(): + # Ignore the warning because we actually don't modify the buffer in this function + warnings.filterwarnings("ignore", message="The given buffer is not writable") + src = torch.frombuffer(src, dtype=torch.uint8) + elif isinstance(src, torch.Tensor): + if self.backend in ["cuda", "pyav"]: + raise RuntimeError( + "VideoReader cannot be initialized from Tensor object when using cuda or pyav backend." + ) + else: + raise ValueError(f"src must be either string, Tensor or bytes object. Got {type(src)}") + + if self.backend == "cuda": + device = torch.device("cuda") + self._c = torch.classes.torchvision.GPUDecoder(src, device) + + elif self.backend == "video_reader": + if isinstance(src, str): + self._c = torch.classes.torchvision.Video(src, stream, num_threads) + elif isinstance(src, torch.Tensor): + self._c = torch.classes.torchvision.Video("", "", 0) + self._c.init_from_memory(src, stream, num_threads) + + elif self.backend == "pyav": + self.container = av.open(src, metadata_errors="ignore") + # TODO: load metadata + stream_type = stream.split(":")[0] + stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1]) + self.pyav_stream = {stream_type: stream_id} + self._c = self.container.decode(**self.pyav_stream) + + # TODO: add extradata exception + + else: + raise RuntimeError("Unknown video backend: {}".format(self.backend)) + + def __next__(self) -> Dict[str, Any]: + """Decodes and returns the next frame of the current stream. + Frames are encoded as a dict with mandatory + data and pts fields, where data is a tensor, and pts is a + presentation timestamp of the frame expressed in seconds + as a float. + + Returns: + (dict): a dictionary and containing decoded frame (``data``) + and corresponding timestamp (``pts``) in seconds + + """ + if self.backend == "cuda": + frame = self._c.next() + if frame.numel() == 0: + raise StopIteration + return {"data": frame, "pts": None} + elif self.backend == "video_reader": + frame, pts = self._c.next() + else: + try: + frame = next(self._c) + pts = float(frame.pts * frame.time_base) + if "video" in self.pyav_stream: + frame = torch.as_tensor(frame.to_rgb().to_ndarray()).permute(2, 0, 1) + elif "audio" in self.pyav_stream: + frame = torch.as_tensor(frame.to_ndarray()).permute(1, 0) + else: + frame = None + except av.error.EOFError: + raise StopIteration + + if frame.numel() == 0: + raise StopIteration + + return {"data": frame, "pts": pts} + + def __iter__(self) -> Iterator[Dict[str, Any]]: + return self + + def seek(self, time_s: float, keyframes_only: bool = False) -> "VideoReader": + """Seek within current stream. + + Args: + time_s (float): seek time in seconds + keyframes_only (bool): allow to seek only to keyframes + + .. note:: + Current implementation is the so-called precise seek. This + means following seek, call to :mod:`next()` will return the + frame with the exact timestamp if it exists or + the first frame with timestamp larger than ``time_s``. + """ + if self.backend in ["cuda", "video_reader"]: + self._c.seek(time_s, keyframes_only) + else: + # handle special case as pyav doesn't catch it + if time_s < 0: + time_s = 0 + temp_str = self.container.streams.get(**self.pyav_stream)[0] + offset = int(round(time_s / temp_str.time_base)) + if not keyframes_only: + warnings.warn("Accurate seek is not implemented for pyav backend") + self.container.seek(offset, backward=True, any_frame=False, stream=temp_str) + self._c = self.container.decode(**self.pyav_stream) + return self + + def get_metadata(self) -> Dict[str, Any]: + """Returns video metadata + + Returns: + (dict): dictionary containing duration and frame rate for every stream + """ + if self.backend == "pyav": + metadata = {} # type: Dict[str, Any] + for stream in self.container.streams: + if stream.type not in metadata: + if stream.type == "video": + rate_n = "fps" + else: + rate_n = "framerate" + metadata[stream.type] = {rate_n: [], "duration": []} + + rate = getattr(stream, "average_rate", None) or stream.sample_rate + + metadata[stream.type]["duration"].append(float(stream.duration * stream.time_base)) + metadata[stream.type][rate_n].append(float(rate)) + return metadata + return self._c.get_metadata() + + def set_current_stream(self, stream: str) -> bool: + """Set current stream. + Explicitly define the stream we are operating on. + + Args: + stream (string): descriptor of the required stream. Defaults to ``"video:0"`` + Currently available stream types include ``['video', 'audio']``. + Each descriptor consists of two parts: stream type (e.g. 'video') and + a unique stream id (which are determined by video encoding). + In this way, if the video container contains multiple + streams of the same type, users can access the one they want. + If only stream type is passed, the decoder auto-detects first stream + of that type and returns it. + + Returns: + (bool): True on success, False otherwise + """ + if self.backend == "cuda": + warnings.warn("GPU decoding only works with video stream.") + if self.backend == "pyav": + stream_type = stream.split(":")[0] + stream_id = 0 if len(stream.split(":")) == 1 else int(stream.split(":")[1]) + self.pyav_stream = {stream_type: stream_id} + self._c = self.container.decode(**self.pyav_stream) + return True + return self._c.set_current_stream(stream) diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__init__.py b/lib/python3.12/site-packages/torchvision/tv_tensors/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..1ba47f60a36ad3be8cb2f557adb57f1d2f1ba470 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/__init__.py @@ -0,0 +1,35 @@ +import torch + +from ._bounding_boxes import BoundingBoxes, BoundingBoxFormat +from ._image import Image +from ._mask import Mask +from ._torch_function_helpers import set_return_type +from ._tv_tensor import TVTensor +from ._video import Video + + +# TODO: Fix this. We skip this method as it leads to +# RecursionError: maximum recursion depth exceeded while calling a Python object +# Until `disable` is removed, there will be graph breaks after all calls to functional transforms +@torch.compiler.disable +def wrap(wrappee, *, like, **kwargs): + """Convert a :class:`torch.Tensor` (``wrappee``) into the same :class:`~torchvision.tv_tensors.TVTensor` subclass as ``like``. + + If ``like`` is a :class:`~torchvision.tv_tensors.BoundingBoxes`, the ``format`` and ``canvas_size`` of + ``like`` are assigned to ``wrappee``, unless they are passed as ``kwargs``. + + Args: + wrappee (Tensor): The tensor to convert. + like (:class:`~torchvision.tv_tensors.TVTensor`): The reference. + ``wrappee`` will be converted into the same subclass as ``like``. + kwargs: Can contain "format" and "canvas_size" if ``like`` is a :class:`~torchvision.tv_tensor.BoundingBoxes`. + Ignored otherwise. + """ + if isinstance(like, BoundingBoxes): + return BoundingBoxes._wrap( + wrappee, + format=kwargs.get("format", like.format), + canvas_size=kwargs.get("canvas_size", like.canvas_size), + ) + else: + return wrappee.as_subclass(type(like)) diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..98ed0e823d8fde8b740eb9ceda5943ed06aa7ea0 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_bounding_boxes.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_bounding_boxes.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..329bd2848c6b284bcdc248a341d2bad0688bb465 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_bounding_boxes.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_dataset_wrapper.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_dataset_wrapper.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..7a10525f978a6e4837baad617af4f43a6f9e4be8 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_dataset_wrapper.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_image.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_image.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b7c8466879e3065d4ff330fb97fb4cd0e8009a67 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_image.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_mask.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_mask.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..96621a15536612acc0eb416a92b787b9fc93ece8 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_mask.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_torch_function_helpers.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_torch_function_helpers.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..226230028345f88ef052800a8e4c04f0a5bea155 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_torch_function_helpers.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_tv_tensor.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_tv_tensor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2cf9113b9680cad1141d91325bfca842e130ed03 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_tv_tensor.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_video.cpython-312.pyc b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_video.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..0be8209e59c2e013390c4dd2a2f0b534a2666e02 Binary files /dev/null and b/lib/python3.12/site-packages/torchvision/tv_tensors/__pycache__/_video.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/_bounding_boxes.py b/lib/python3.12/site-packages/torchvision/tv_tensors/_bounding_boxes.py new file mode 100644 index 0000000000000000000000000000000000000000..ea02fa3dc7b3e25ba7f545ca64288fc241d47f2a --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/_bounding_boxes.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +from enum import Enum +from typing import Any, Mapping, Optional, Sequence, Tuple, Union + +import torch +from torch.utils._pytree import tree_flatten + +from ._tv_tensor import TVTensor + + +class BoundingBoxFormat(Enum): + """Coordinate format of a bounding box. + + Available formats are + + * ``XYXY`` + * ``XYWH`` + * ``CXCYWH`` + """ + + XYXY = "XYXY" + XYWH = "XYWH" + CXCYWH = "CXCYWH" + + +class BoundingBoxes(TVTensor): + """:class:`torch.Tensor` subclass for bounding boxes with shape ``[N, 4]``. + + .. note:: + There should be only one :class:`~torchvision.tv_tensors.BoundingBoxes` + instance per sample e.g. ``{"img": img, "bbox": BoundingBoxes(...)}``, + although one :class:`~torchvision.tv_tensors.BoundingBoxes` object can + contain multiple bounding boxes. + + Args: + data: Any data that can be turned into a tensor with :func:`torch.as_tensor`. + format (BoundingBoxFormat, str): Format of the bounding box. + canvas_size (two-tuple of ints): Height and width of the corresponding image or video. + dtype (torch.dtype, optional): Desired data type of the bounding box. If omitted, will be inferred from + ``data``. + device (torch.device, optional): Desired device of the bounding box. If omitted and ``data`` is a + :class:`torch.Tensor`, the device is taken from it. Otherwise, the bounding box is constructed on the CPU. + requires_grad (bool, optional): Whether autograd should record operations on the bounding box. If omitted and + ``data`` is a :class:`torch.Tensor`, the value is taken from it. Otherwise, defaults to ``False``. + """ + + format: BoundingBoxFormat + canvas_size: Tuple[int, int] + + @classmethod + def _wrap(cls, tensor: torch.Tensor, *, format: Union[BoundingBoxFormat, str], canvas_size: Tuple[int, int], check_dims: bool = True) -> BoundingBoxes: # type: ignore[override] + if check_dims: + if tensor.ndim == 1: + tensor = tensor.unsqueeze(0) + elif tensor.ndim != 2: + raise ValueError(f"Expected a 1D or 2D tensor, got {tensor.ndim}D") + if isinstance(format, str): + format = BoundingBoxFormat[format.upper()] + bounding_boxes = tensor.as_subclass(cls) + bounding_boxes.format = format + bounding_boxes.canvas_size = canvas_size + return bounding_boxes + + def __new__( + cls, + data: Any, + *, + format: Union[BoundingBoxFormat, str], + canvas_size: Tuple[int, int], + dtype: Optional[torch.dtype] = None, + device: Optional[Union[torch.device, str, int]] = None, + requires_grad: Optional[bool] = None, + ) -> BoundingBoxes: + tensor = cls._to_tensor(data, dtype=dtype, device=device, requires_grad=requires_grad) + return cls._wrap(tensor, format=format, canvas_size=canvas_size) + + @classmethod + def _wrap_output( + cls, + output: torch.Tensor, + args: Sequence[Any] = (), + kwargs: Optional[Mapping[str, Any]] = None, + ) -> BoundingBoxes: + # If there are BoundingBoxes instances in the output, their metadata got lost when we called + # super().__torch_function__. We need to restore the metadata somehow, so we choose to take + # the metadata from the first bbox in the parameters. + # This should be what we want in most cases. When it's not, it's probably a mis-use anyway, e.g. + # something like some_xyxy_bbox + some_xywh_bbox; we don't guard against those cases. + flat_params, _ = tree_flatten(args + (tuple(kwargs.values()) if kwargs else ())) # type: ignore[operator] + first_bbox_from_args = next(x for x in flat_params if isinstance(x, BoundingBoxes)) + format, canvas_size = first_bbox_from_args.format, first_bbox_from_args.canvas_size + + if isinstance(output, torch.Tensor) and not isinstance(output, BoundingBoxes): + output = BoundingBoxes._wrap(output, format=format, canvas_size=canvas_size, check_dims=False) + elif isinstance(output, (tuple, list)): + output = type(output)( + BoundingBoxes._wrap(part, format=format, canvas_size=canvas_size, check_dims=False) for part in output + ) + return output + + def __repr__(self, *, tensor_contents: Any = None) -> str: # type: ignore[override] + return self._make_repr(format=self.format, canvas_size=self.canvas_size) diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/_dataset_wrapper.py b/lib/python3.12/site-packages/torchvision/tv_tensors/_dataset_wrapper.py new file mode 100644 index 0000000000000000000000000000000000000000..23683221f6005a9ce6a55e785e59409a649d7928 --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/_dataset_wrapper.py @@ -0,0 +1,666 @@ +# type: ignore + +from __future__ import annotations + +import collections.abc + +import contextlib +from collections import defaultdict +from copy import copy + +import torch + +from torchvision import datasets, tv_tensors +from torchvision.transforms.v2 import functional as F + +__all__ = ["wrap_dataset_for_transforms_v2"] + + +def wrap_dataset_for_transforms_v2(dataset, target_keys=None): + """Wrap a ``torchvision.dataset`` for usage with :mod:`torchvision.transforms.v2`. + + Example: + >>> dataset = torchvision.datasets.CocoDetection(...) + >>> dataset = wrap_dataset_for_transforms_v2(dataset) + + .. note:: + + For now, only the most popular datasets are supported. Furthermore, the wrapper only supports dataset + configurations that are fully supported by ``torchvision.transforms.v2``. If you encounter an error prompting you + to raise an issue to ``torchvision`` for a dataset or configuration that you need, please do so. + + The dataset samples are wrapped according to the description below. + + Special cases: + + * :class:`~torchvision.datasets.CocoDetection`: Instead of returning the target as list of dicts, the wrapper + returns a dict of lists. In addition, the key-value-pairs ``"boxes"`` (in ``XYXY`` coordinate format), + ``"masks"`` and ``"labels"`` are added and wrap the data in the corresponding ``torchvision.tv_tensors``. + The original keys are preserved. If ``target_keys`` is omitted, returns only the values for the + ``"image_id"``, ``"boxes"``, and ``"labels"``. + * :class:`~torchvision.datasets.VOCDetection`: The key-value-pairs ``"boxes"`` and ``"labels"`` are added to + the target and wrap the data in the corresponding ``torchvision.tv_tensors``. The original keys are + preserved. If ``target_keys`` is omitted, returns only the values for the ``"boxes"`` and ``"labels"``. + * :class:`~torchvision.datasets.CelebA`: The target for ``target_type="bbox"`` is converted to the ``XYXY`` + coordinate format and wrapped into a :class:`~torchvision.tv_tensors.BoundingBoxes` tv_tensor. + * :class:`~torchvision.datasets.Kitti`: Instead returning the target as list of dicts, the wrapper returns a + dict of lists. In addition, the key-value-pairs ``"boxes"`` and ``"labels"`` are added and wrap the data + in the corresponding ``torchvision.tv_tensors``. The original keys are preserved. If ``target_keys`` is + omitted, returns only the values for the ``"boxes"`` and ``"labels"``. + * :class:`~torchvision.datasets.OxfordIIITPet`: The target for ``target_type="segmentation"`` is wrapped into a + :class:`~torchvision.tv_tensors.Mask` tv_tensor. + * :class:`~torchvision.datasets.Cityscapes`: The target for ``target_type="semantic"`` is wrapped into a + :class:`~torchvision.tv_tensors.Mask` tv_tensor. The target for ``target_type="instance"`` is *replaced* by + a dictionary with the key-value-pairs ``"masks"`` (as :class:`~torchvision.tv_tensors.Mask` tv_tensor) and + ``"labels"``. + * :class:`~torchvision.datasets.WIDERFace`: The value for key ``"bbox"`` in the target is converted to ``XYXY`` + coordinate format and wrapped into a :class:`~torchvision.tv_tensors.BoundingBoxes` tv_tensor. + + Image classification datasets + + This wrapper is a no-op for image classification datasets, since they were already fully supported by + :mod:`torchvision.transforms` and thus no change is needed for :mod:`torchvision.transforms.v2`. + + Segmentation datasets + + Segmentation datasets, e.g. :class:`~torchvision.datasets.VOCSegmentation`, return a two-tuple of + :class:`PIL.Image.Image`'s. This wrapper leaves the image as is (first item), while wrapping the + segmentation mask into a :class:`~torchvision.tv_tensors.Mask` (second item). + + Video classification datasets + + Video classification datasets, e.g. :class:`~torchvision.datasets.Kinetics`, return a three-tuple containing a + :class:`torch.Tensor` for the video and audio and a :class:`int` as label. This wrapper wraps the video into a + :class:`~torchvision.tv_tensors.Video` while leaving the other items as is. + + .. note:: + + Only datasets constructed with ``output_format="TCHW"`` are supported, since the alternative + ``output_format="THWC"`` is not supported by :mod:`torchvision.transforms.v2`. + + Args: + dataset: the dataset instance to wrap for compatibility with transforms v2. + target_keys: Target keys to return in case the target is a dictionary. If ``None`` (default), selected keys are + specific to the dataset. If ``"all"``, returns the full target. Can also be a collection of strings for + fine grained access. Currently only supported for :class:`~torchvision.datasets.CocoDetection`, + :class:`~torchvision.datasets.VOCDetection`, :class:`~torchvision.datasets.Kitti`, and + :class:`~torchvision.datasets.WIDERFace`. See above for details. + """ + if not ( + target_keys is None + or target_keys == "all" + or (isinstance(target_keys, collections.abc.Collection) and all(isinstance(key, str) for key in target_keys)) + ): + raise ValueError( + f"`target_keys` can be None, 'all', or a collection of strings denoting the keys to be returned, " + f"but got {target_keys}" + ) + + # Imagine we have isinstance(dataset, datasets.ImageNet). This will create a new class with the name + # "WrappedImageNet" at runtime that doubly inherits from VisionDatasetTVTensorWrapper (see below) as well as the + # original ImageNet class. This allows the user to do regular isinstance(wrapped_dataset, datasets.ImageNet) checks, + # while we can still inject everything that we need. + wrapped_dataset_cls = type(f"Wrapped{type(dataset).__name__}", (VisionDatasetTVTensorWrapper, type(dataset)), {}) + # Since VisionDatasetTVTensorWrapper comes before ImageNet in the MRO, calling the class hits + # VisionDatasetTVTensorWrapper.__init__ first. Since we are never doing super().__init__(...), the constructor of + # ImageNet is never hit. That is by design, since we don't want to create the dataset instance again, but rather + # have the existing instance as attribute on the new object. + return wrapped_dataset_cls(dataset, target_keys) + + +class WrapperFactories(dict): + def register(self, dataset_cls): + def decorator(wrapper_factory): + self[dataset_cls] = wrapper_factory + return wrapper_factory + + return decorator + + +# We need this two-stage design, i.e. a wrapper factory producing the actual wrapper, since some wrappers depend on the +# dataset instance rather than just the class, since they require the user defined instance attributes. Thus, we can +# provide a wrapping from the dataset class to the factory here, but can only instantiate the wrapper at runtime when +# we have access to the dataset instance. +WRAPPER_FACTORIES = WrapperFactories() + + +class VisionDatasetTVTensorWrapper: + def __init__(self, dataset, target_keys): + dataset_cls = type(dataset) + + if not isinstance(dataset, datasets.VisionDataset): + raise TypeError( + f"This wrapper is meant for subclasses of `torchvision.datasets.VisionDataset`, " + f"but got a '{dataset_cls.__name__}' instead.\n" + f"For an example of how to perform the wrapping for custom datasets, see\n\n" + "https://pytorch.org/vision/main/auto_examples/plot_tv_tensors.html#do-i-have-to-wrap-the-output-of-the-datasets-myself" + ) + + for cls in dataset_cls.mro(): + if cls in WRAPPER_FACTORIES: + wrapper_factory = WRAPPER_FACTORIES[cls] + if target_keys is not None and cls not in { + datasets.CocoDetection, + datasets.VOCDetection, + datasets.Kitti, + datasets.WIDERFace, + }: + raise ValueError( + f"`target_keys` is currently only supported for `CocoDetection`, `VOCDetection`, `Kitti`, " + f"and `WIDERFace`, but got {cls.__name__}." + ) + break + elif cls is datasets.VisionDataset: + # TODO: If we have documentation on how to do that, put a link in the error message. + msg = f"No wrapper exists for dataset class {dataset_cls.__name__}. Please wrap the output yourself." + if dataset_cls in datasets.__dict__.values(): + msg = ( + f"{msg} If an automated wrapper for this dataset would be useful for you, " + f"please open an issue at https://github.com/pytorch/vision/issues." + ) + raise TypeError(msg) + + self._dataset = dataset + self._target_keys = target_keys + self._wrapper = wrapper_factory(dataset, target_keys) + + # We need to disable the transforms on the dataset here to be able to inject the wrapping before we apply them. + # Although internally, `datasets.VisionDataset` merges `transform` and `target_transform` into the joint + # `transforms` + # https://github.com/pytorch/vision/blob/135a0f9ea9841b6324b4fe8974e2543cbb95709a/torchvision/datasets/vision.py#L52-L54 + # some (if not most) datasets still use `transform` and `target_transform` individually. Thus, we need to + # disable all three here to be able to extract the untransformed sample to wrap. + self.transform, dataset.transform = dataset.transform, None + self.target_transform, dataset.target_transform = dataset.target_transform, None + self.transforms, dataset.transforms = dataset.transforms, None + + def __getattr__(self, item): + with contextlib.suppress(AttributeError): + return object.__getattribute__(self, item) + + return getattr(self._dataset, item) + + def __getitem__(self, idx): + # This gets us the raw sample since we disabled the transforms for the underlying dataset in the constructor + # of this class + sample = self._dataset[idx] + + sample = self._wrapper(idx, sample) + + # Regardless of whether the user has supplied the transforms individually (`transform` and `target_transform`) + # or joint (`transforms`), we can access the full functionality through `transforms` + if self.transforms is not None: + sample = self.transforms(*sample) + + return sample + + def __len__(self): + return len(self._dataset) + + # TODO: maybe we should use __getstate__ and __setstate__ instead of __reduce__, as recommended in the docs. + def __reduce__(self): + # __reduce__ gets called when we try to pickle the dataset. + # In a DataLoader with spawn context, this gets called `num_workers` times from the main process. + + # We have to reset the [target_]transform[s] attributes of the dataset + # to their original values, because we previously set them to None in __init__(). + dataset = copy(self._dataset) + dataset.transform = self.transform + dataset.transforms = self.transforms + dataset.target_transform = self.target_transform + + return wrap_dataset_for_transforms_v2, (dataset, self._target_keys) + + +def raise_not_supported(description): + raise RuntimeError( + f"{description} is currently not supported by this wrapper. " + f"If this would be helpful for you, please open an issue at https://github.com/pytorch/vision/issues." + ) + + +def identity(item): + return item + + +def identity_wrapper_factory(dataset, target_keys): + def wrapper(idx, sample): + return sample + + return wrapper + + +def pil_image_to_mask(pil_image): + return tv_tensors.Mask(pil_image) + + +def parse_target_keys(target_keys, *, available, default): + if target_keys is None: + target_keys = default + if target_keys == "all": + target_keys = available + else: + target_keys = set(target_keys) + extra = target_keys - available + if extra: + raise ValueError(f"Target keys {sorted(extra)} are not available") + + return target_keys + + +def list_of_dicts_to_dict_of_lists(list_of_dicts): + dict_of_lists = defaultdict(list) + for dct in list_of_dicts: + for key, value in dct.items(): + dict_of_lists[key].append(value) + return dict(dict_of_lists) + + +def wrap_target_by_type(target, *, target_types, type_wrappers): + if not isinstance(target, (tuple, list)): + target = [target] + + wrapped_target = tuple( + type_wrappers.get(target_type, identity)(item) for target_type, item in zip(target_types, target) + ) + + if len(wrapped_target) == 1: + wrapped_target = wrapped_target[0] + + return wrapped_target + + +def classification_wrapper_factory(dataset, target_keys): + return identity_wrapper_factory(dataset, target_keys) + + +for dataset_cls in [ + datasets.Caltech256, + datasets.CIFAR10, + datasets.CIFAR100, + datasets.ImageNet, + datasets.MNIST, + datasets.FashionMNIST, + datasets.GTSRB, + datasets.DatasetFolder, + datasets.ImageFolder, + datasets.Imagenette, +]: + WRAPPER_FACTORIES.register(dataset_cls)(classification_wrapper_factory) + + +def segmentation_wrapper_factory(dataset, target_keys): + def wrapper(idx, sample): + image, mask = sample + return image, pil_image_to_mask(mask) + + return wrapper + + +for dataset_cls in [ + datasets.VOCSegmentation, +]: + WRAPPER_FACTORIES.register(dataset_cls)(segmentation_wrapper_factory) + + +def video_classification_wrapper_factory(dataset, target_keys): + if dataset.video_clips.output_format == "THWC": + raise RuntimeError( + f"{type(dataset).__name__} with `output_format='THWC'` is not supported by this wrapper, " + f"since it is not compatible with the transformations. Please use `output_format='TCHW'` instead." + ) + + def wrapper(idx, sample): + video, audio, label = sample + + video = tv_tensors.Video(video) + + return video, audio, label + + return wrapper + + +for dataset_cls in [ + datasets.HMDB51, + datasets.Kinetics, + datasets.UCF101, +]: + WRAPPER_FACTORIES.register(dataset_cls)(video_classification_wrapper_factory) + + +@WRAPPER_FACTORIES.register(datasets.Caltech101) +def caltech101_wrapper_factory(dataset, target_keys): + if "annotation" in dataset.target_type: + raise_not_supported("Caltech101 dataset with `target_type=['annotation', ...]`") + + return classification_wrapper_factory(dataset, target_keys) + + +@WRAPPER_FACTORIES.register(datasets.CocoDetection) +def coco_dectection_wrapper_factory(dataset, target_keys): + target_keys = parse_target_keys( + target_keys, + available={ + # native + "segmentation", + "area", + "iscrowd", + "image_id", + "bbox", + "category_id", + # added by the wrapper + "boxes", + "masks", + "labels", + }, + default={"image_id", "boxes", "labels"}, + ) + + def segmentation_to_mask(segmentation, *, canvas_size): + from pycocotools import mask + + if isinstance(segmentation, dict): + # if counts is a string, it is already an encoded RLE mask + if not isinstance(segmentation["counts"], str): + segmentation = mask.frPyObjects(segmentation, *canvas_size) + elif isinstance(segmentation, list): + segmentation = mask.merge(mask.frPyObjects(segmentation, *canvas_size)) + else: + raise ValueError(f"COCO segmentation expected to be a dict or a list, got {type(segmentation)}") + return torch.from_numpy(mask.decode(segmentation)) + + def wrapper(idx, sample): + image_id = dataset.ids[idx] + + image, target = sample + + if not target: + return image, dict(image_id=image_id) + + canvas_size = tuple(F.get_size(image)) + + batched_target = list_of_dicts_to_dict_of_lists(target) + target = {} + + if "image_id" in target_keys: + target["image_id"] = image_id + + if "boxes" in target_keys: + target["boxes"] = F.convert_bounding_box_format( + tv_tensors.BoundingBoxes( + batched_target["bbox"], + format=tv_tensors.BoundingBoxFormat.XYWH, + canvas_size=canvas_size, + ), + new_format=tv_tensors.BoundingBoxFormat.XYXY, + ) + + if "masks" in target_keys: + target["masks"] = tv_tensors.Mask( + torch.stack( + [ + segmentation_to_mask(segmentation, canvas_size=canvas_size) + for segmentation in batched_target["segmentation"] + ] + ), + ) + + if "labels" in target_keys: + target["labels"] = torch.tensor(batched_target["category_id"]) + + for target_key in target_keys - {"image_id", "boxes", "masks", "labels"}: + target[target_key] = batched_target[target_key] + + return image, target + + return wrapper + + +WRAPPER_FACTORIES.register(datasets.CocoCaptions)(identity_wrapper_factory) + + +VOC_DETECTION_CATEGORIES = [ + "__background__", + "aeroplane", + "bicycle", + "bird", + "boat", + "bottle", + "bus", + "car", + "cat", + "chair", + "cow", + "diningtable", + "dog", + "horse", + "motorbike", + "person", + "pottedplant", + "sheep", + "sofa", + "train", + "tvmonitor", +] +VOC_DETECTION_CATEGORY_TO_IDX = dict(zip(VOC_DETECTION_CATEGORIES, range(len(VOC_DETECTION_CATEGORIES)))) + + +@WRAPPER_FACTORIES.register(datasets.VOCDetection) +def voc_detection_wrapper_factory(dataset, target_keys): + target_keys = parse_target_keys( + target_keys, + available={ + # native + "annotation", + # added by the wrapper + "boxes", + "labels", + }, + default={"boxes", "labels"}, + ) + + def wrapper(idx, sample): + image, target = sample + + batched_instances = list_of_dicts_to_dict_of_lists(target["annotation"]["object"]) + + if "annotation" not in target_keys: + target = {} + + if "boxes" in target_keys: + target["boxes"] = tv_tensors.BoundingBoxes( + [ + [int(bndbox[part]) for part in ("xmin", "ymin", "xmax", "ymax")] + for bndbox in batched_instances["bndbox"] + ], + format=tv_tensors.BoundingBoxFormat.XYXY, + canvas_size=(image.height, image.width), + ) + + if "labels" in target_keys: + target["labels"] = torch.tensor( + [VOC_DETECTION_CATEGORY_TO_IDX[category] for category in batched_instances["name"]] + ) + + return image, target + + return wrapper + + +@WRAPPER_FACTORIES.register(datasets.SBDataset) +def sbd_wrapper(dataset, target_keys): + if dataset.mode == "boundaries": + raise_not_supported("SBDataset with mode='boundaries'") + + return segmentation_wrapper_factory(dataset, target_keys) + + +@WRAPPER_FACTORIES.register(datasets.CelebA) +def celeba_wrapper_factory(dataset, target_keys): + if any(target_type in dataset.target_type for target_type in ["attr", "landmarks"]): + raise_not_supported("`CelebA` dataset with `target_type=['attr', 'landmarks', ...]`") + + def wrapper(idx, sample): + image, target = sample + + target = wrap_target_by_type( + target, + target_types=dataset.target_type, + type_wrappers={ + "bbox": lambda item: F.convert_bounding_box_format( + tv_tensors.BoundingBoxes( + item, + format=tv_tensors.BoundingBoxFormat.XYWH, + canvas_size=(image.height, image.width), + ), + new_format=tv_tensors.BoundingBoxFormat.XYXY, + ), + }, + ) + + return image, target + + return wrapper + + +KITTI_CATEGORIES = ["Car", "Van", "Truck", "Pedestrian", "Person_sitting", "Cyclist", "Tram", "Misc", "DontCare"] +KITTI_CATEGORY_TO_IDX = dict(zip(KITTI_CATEGORIES, range(len(KITTI_CATEGORIES)))) + + +@WRAPPER_FACTORIES.register(datasets.Kitti) +def kitti_wrapper_factory(dataset, target_keys): + target_keys = parse_target_keys( + target_keys, + available={ + # native + "type", + "truncated", + "occluded", + "alpha", + "bbox", + "dimensions", + "location", + "rotation_y", + # added by the wrapper + "boxes", + "labels", + }, + default={"boxes", "labels"}, + ) + + def wrapper(idx, sample): + image, target = sample + + if target is None: + return image, target + + batched_target = list_of_dicts_to_dict_of_lists(target) + target = {} + + if "boxes" in target_keys: + target["boxes"] = tv_tensors.BoundingBoxes( + batched_target["bbox"], + format=tv_tensors.BoundingBoxFormat.XYXY, + canvas_size=(image.height, image.width), + ) + + if "labels" in target_keys: + target["labels"] = torch.tensor([KITTI_CATEGORY_TO_IDX[category] for category in batched_target["type"]]) + + for target_key in target_keys - {"boxes", "labels"}: + target[target_key] = batched_target[target_key] + + return image, target + + return wrapper + + +@WRAPPER_FACTORIES.register(datasets.OxfordIIITPet) +def oxford_iiit_pet_wrapper_factor(dataset, target_keys): + def wrapper(idx, sample): + image, target = sample + + if target is not None: + target = wrap_target_by_type( + target, + target_types=dataset._target_types, + type_wrappers={ + "segmentation": pil_image_to_mask, + }, + ) + + return image, target + + return wrapper + + +@WRAPPER_FACTORIES.register(datasets.Cityscapes) +def cityscapes_wrapper_factory(dataset, target_keys): + if any(target_type in dataset.target_type for target_type in ["polygon", "color"]): + raise_not_supported("`Cityscapes` dataset with `target_type=['polygon', 'color', ...]`") + + def instance_segmentation_wrapper(mask): + # See https://github.com/mcordts/cityscapesScripts/blob/8da5dd00c9069058ccc134654116aac52d4f6fa2/cityscapesscripts/preparation/json2instanceImg.py#L7-L21 + data = pil_image_to_mask(mask) + masks = [] + labels = [] + for id in data.unique(): + masks.append(data == id) + label = id + if label >= 1_000: + label //= 1_000 + labels.append(label) + return dict(masks=tv_tensors.Mask(torch.stack(masks)), labels=torch.stack(labels)) + + def wrapper(idx, sample): + image, target = sample + + target = wrap_target_by_type( + target, + target_types=dataset.target_type, + type_wrappers={ + "instance": instance_segmentation_wrapper, + "semantic": pil_image_to_mask, + }, + ) + + return image, target + + return wrapper + + +@WRAPPER_FACTORIES.register(datasets.WIDERFace) +def widerface_wrapper(dataset, target_keys): + target_keys = parse_target_keys( + target_keys, + available={ + "bbox", + "blur", + "expression", + "illumination", + "occlusion", + "pose", + "invalid", + }, + default="all", + ) + + def wrapper(idx, sample): + image, target = sample + + if target is None: + return image, target + + target = {key: target[key] for key in target_keys} + + if "bbox" in target_keys: + target["bbox"] = F.convert_bounding_box_format( + tv_tensors.BoundingBoxes( + target["bbox"], format=tv_tensors.BoundingBoxFormat.XYWH, canvas_size=(image.height, image.width) + ), + new_format=tv_tensors.BoundingBoxFormat.XYXY, + ) + + return image, target + + return wrapper diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/_image.py b/lib/python3.12/site-packages/torchvision/tv_tensors/_image.py new file mode 100644 index 0000000000000000000000000000000000000000..2a0a2ec720966f849f8d832a1b9f2e640ba7dc2c --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/_image.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from typing import Any, Optional, Union + +import PIL.Image +import torch + +from ._tv_tensor import TVTensor + + +class Image(TVTensor): + """:class:`torch.Tensor` subclass for images with shape ``[..., C, H, W]``. + + .. note:: + + In the :ref:`transforms `, ``Image`` instances are largely + interchangeable with pure :class:`torch.Tensor`. See + :ref:`this note ` for more details. + + Args: + data (tensor-like, PIL.Image.Image): Any data that can be turned into a tensor with :func:`torch.as_tensor` as + well as PIL images. + dtype (torch.dtype, optional): Desired data type. If omitted, will be inferred from + ``data``. + device (torch.device, optional): Desired device. If omitted and ``data`` is a + :class:`torch.Tensor`, the device is taken from it. Otherwise, the image is constructed on the CPU. + requires_grad (bool, optional): Whether autograd should record operations. If omitted and + ``data`` is a :class:`torch.Tensor`, the value is taken from it. Otherwise, defaults to ``False``. + """ + + def __new__( + cls, + data: Any, + *, + dtype: Optional[torch.dtype] = None, + device: Optional[Union[torch.device, str, int]] = None, + requires_grad: Optional[bool] = None, + ) -> Image: + if isinstance(data, PIL.Image.Image): + from torchvision.transforms.v2 import functional as F + + data = F.pil_to_tensor(data) + + tensor = cls._to_tensor(data, dtype=dtype, device=device, requires_grad=requires_grad) + if tensor.ndim < 2: + raise ValueError + elif tensor.ndim == 2: + tensor = tensor.unsqueeze(0) + + return tensor.as_subclass(cls) + + def __repr__(self, *, tensor_contents: Any = None) -> str: # type: ignore[override] + return self._make_repr() diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/_mask.py b/lib/python3.12/site-packages/torchvision/tv_tensors/_mask.py new file mode 100644 index 0000000000000000000000000000000000000000..ef9d96159fb5e0ac64bb99774ae65be9e11325ae --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/_mask.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from typing import Any, Optional, Union + +import PIL.Image +import torch + +from ._tv_tensor import TVTensor + + +class Mask(TVTensor): + """:class:`torch.Tensor` subclass for segmentation and detection masks with shape ``[..., H, W]``. + + Args: + data (tensor-like, PIL.Image.Image): Any data that can be turned into a tensor with :func:`torch.as_tensor` as + well as PIL images. + dtype (torch.dtype, optional): Desired data type. If omitted, will be inferred from + ``data``. + device (torch.device, optional): Desired device. If omitted and ``data`` is a + :class:`torch.Tensor`, the device is taken from it. Otherwise, the mask is constructed on the CPU. + requires_grad (bool, optional): Whether autograd should record operations. If omitted and + ``data`` is a :class:`torch.Tensor`, the value is taken from it. Otherwise, defaults to ``False``. + """ + + def __new__( + cls, + data: Any, + *, + dtype: Optional[torch.dtype] = None, + device: Optional[Union[torch.device, str, int]] = None, + requires_grad: Optional[bool] = None, + ) -> Mask: + if isinstance(data, PIL.Image.Image): + from torchvision.transforms.v2 import functional as F + + data = F.pil_to_tensor(data) + + tensor = cls._to_tensor(data, dtype=dtype, device=device, requires_grad=requires_grad) + return tensor.as_subclass(cls) diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/_torch_function_helpers.py b/lib/python3.12/site-packages/torchvision/tv_tensors/_torch_function_helpers.py new file mode 100644 index 0000000000000000000000000000000000000000..e6ea5fddf35b4b585cb2527863fc2db1ba6c049b --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/_torch_function_helpers.py @@ -0,0 +1,72 @@ +import torch + +_TORCHFUNCTION_SUBCLASS = False + + +class _ReturnTypeCM: + def __init__(self, to_restore): + self.to_restore = to_restore + + def __enter__(self): + return self + + def __exit__(self, *args): + global _TORCHFUNCTION_SUBCLASS + _TORCHFUNCTION_SUBCLASS = self.to_restore + + +def set_return_type(return_type: str): + """Set the return type of torch operations on :class:`~torchvision.tv_tensors.TVTensor`. + + This only affects the behaviour of torch operations. It has no effect on + ``torchvision`` transforms or functionals, which will always return as + output the same type that was passed as input. + + .. warning:: + + We recommend using :class:`~torchvision.transforms.v2.ToPureTensor` at + the end of your transform pipelines if you use + ``set_return_type("TVTensor")``. This will avoid the + ``__torch_function__`` overhead in the models ``forward()``. + + Can be used as a global flag for the entire program: + + .. code:: python + + img = tv_tensors.Image(torch.rand(3, 5, 5)) + img + 2 # This is a pure Tensor (default behaviour) + + set_return_type("TVTensor") + img + 2 # This is an Image + + or as a context manager to restrict the scope: + + .. code:: python + + img = tv_tensors.Image(torch.rand(3, 5, 5)) + img + 2 # This is a pure Tensor + with set_return_type("TVTensor"): + img + 2 # This is an Image + img + 2 # This is a pure Tensor + + Args: + return_type (str): Can be "TVTensor" or "Tensor" (case-insensitive). + Default is "Tensor" (i.e. pure :class:`torch.Tensor`). + """ + global _TORCHFUNCTION_SUBCLASS + to_restore = _TORCHFUNCTION_SUBCLASS + + try: + _TORCHFUNCTION_SUBCLASS = {"tensor": False, "tvtensor": True}[return_type.lower()] + except KeyError: + raise ValueError(f"return_type must be 'TVTensor' or 'Tensor', got {return_type}") from None + + return _ReturnTypeCM(to_restore) + + +def _must_return_subclass(): + return _TORCHFUNCTION_SUBCLASS + + +# For those ops we always want to preserve the original subclass instead of returning a pure Tensor +_FORCE_TORCHFUNCTION_SUBCLASS = {torch.Tensor.clone, torch.Tensor.to, torch.Tensor.detach, torch.Tensor.requires_grad_} diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/_tv_tensor.py b/lib/python3.12/site-packages/torchvision/tv_tensors/_tv_tensor.py new file mode 100644 index 0000000000000000000000000000000000000000..508e73724beb8e5ce1202bc35e079b5542149feb --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/_tv_tensor.py @@ -0,0 +1,132 @@ +from __future__ import annotations + +from typing import Any, Callable, Dict, Mapping, Optional, Sequence, Tuple, Type, TypeVar, Union + +import torch +from torch._C import DisableTorchFunctionSubclass +from torch.types import _device, _dtype, _size + +from torchvision.tv_tensors._torch_function_helpers import _FORCE_TORCHFUNCTION_SUBCLASS, _must_return_subclass + + +D = TypeVar("D", bound="TVTensor") + + +class TVTensor(torch.Tensor): + """Base class for all TVTensors. + + You probably don't want to use this class unless you're defining your own + custom TVTensors. See + :ref:`sphx_glr_auto_examples_transforms_plot_custom_tv_tensors.py` for details. + """ + + @staticmethod + def _to_tensor( + data: Any, + dtype: Optional[torch.dtype] = None, + device: Optional[Union[torch.device, str, int]] = None, + requires_grad: Optional[bool] = None, + ) -> torch.Tensor: + if requires_grad is None: + requires_grad = data.requires_grad if isinstance(data, torch.Tensor) else False + return torch.as_tensor(data, dtype=dtype, device=device).requires_grad_(requires_grad) + + @classmethod + def _wrap_output( + cls, + output: torch.Tensor, + args: Sequence[Any] = (), + kwargs: Optional[Mapping[str, Any]] = None, + ) -> torch.Tensor: + # Same as torch._tensor._convert + if isinstance(output, torch.Tensor) and not isinstance(output, cls): + output = output.as_subclass(cls) + + if isinstance(output, (tuple, list)): + # Also handles things like namedtuples + output = type(output)(cls._wrap_output(part, args, kwargs) for part in output) + return output + + @classmethod + def __torch_function__( + cls, + func: Callable[..., torch.Tensor], + types: Tuple[Type[torch.Tensor], ...], + args: Sequence[Any] = (), + kwargs: Optional[Mapping[str, Any]] = None, + ) -> torch.Tensor: + """For general information about how the __torch_function__ protocol works, + see https://pytorch.org/docs/stable/notes/extending.html#extending-torch + + TL;DR: Every time a PyTorch operator is called, it goes through the inputs and looks for the + ``__torch_function__`` method. If one is found, it is invoked with the operator as ``func`` as well as the + ``args`` and ``kwargs`` of the original call. + + Why do we override this? Because the base implementation in torch.Tensor would preserve the TVTensor type + of the output. In our case, we want to return pure tensors instead (with a few exceptions). Refer to the + "TVTensors FAQ" gallery example for a rationale of this behaviour (TL;DR: perf + no silver bullet). + + Our implementation below is very similar to the base implementation in ``torch.Tensor`` - go check it out. + """ + if not all(issubclass(cls, t) for t in types): + return NotImplemented + + # Like in the base Tensor.__torch_function__ implementation, it's easier to always use + # DisableTorchFunctionSubclass and then manually re-wrap the output if necessary + with DisableTorchFunctionSubclass(): + output = func(*args, **kwargs or dict()) + + must_return_subclass = _must_return_subclass() + if must_return_subclass or (func in _FORCE_TORCHFUNCTION_SUBCLASS and isinstance(args[0], cls)): + # If you're wondering why we need the `isinstance(args[0], cls)` check, remove it and see what fails + # in test_to_tv_tensor_reference(). + # The __torch_function__ protocol will invoke the __torch_function__ method on *all* types involved in + # the computation by walking the MRO upwards. For example, + # `out = a_pure_tensor.to(an_image)` will invoke `Image.__torch_function__` with + # `args = (a_pure_tensor, an_image)` first. Without this guard, `out` would + # be wrapped into an `Image`. + return cls._wrap_output(output, args, kwargs) + + if not must_return_subclass and isinstance(output, cls): + # DisableTorchFunctionSubclass is ignored by inplace ops like `.add_(...)`, + # so for those, the output is still a TVTensor. Thus, we need to manually unwrap. + return output.as_subclass(torch.Tensor) + + return output + + def _make_repr(self, **kwargs: Any) -> str: + # This is a poor man's implementation of the proposal in https://github.com/pytorch/pytorch/issues/76532. + # If that ever gets implemented, remove this in favor of the solution on the `torch.Tensor` class. + extra_repr = ", ".join(f"{key}={value}" for key, value in kwargs.items()) + return f"{super().__repr__()[:-1]}, {extra_repr})" + + # Add properties for common attributes like shape, dtype, device, ndim etc + # this way we return the result without passing into __torch_function__ + @property + def shape(self) -> _size: # type: ignore[override] + with DisableTorchFunctionSubclass(): + return super().shape + + @property + def ndim(self) -> int: # type: ignore[override] + with DisableTorchFunctionSubclass(): + return super().ndim + + @property + def device(self, *args: Any, **kwargs: Any) -> _device: # type: ignore[override] + with DisableTorchFunctionSubclass(): + return super().device + + @property + def dtype(self) -> _dtype: # type: ignore[override] + with DisableTorchFunctionSubclass(): + return super().dtype + + def __deepcopy__(self: D, memo: Dict[int, Any]) -> D: + # We need to detach first, since a plain `Tensor.clone` will be part of the computation graph, which does + # *not* happen for `deepcopy(Tensor)`. A side-effect from detaching is that the `Tensor.requires_grad` + # attribute is cleared, so we need to refill it before we return. + # Note: We don't explicitly handle deep-copying of the metadata here. The only metadata we currently have is + # `BoundingBoxes.format` and `BoundingBoxes.canvas_size`, which are immutable and thus implicitly deep-copied by + # `BoundingBoxes.clone()`. + return self.detach().clone().requires_grad_(self.requires_grad) # type: ignore[return-value] diff --git a/lib/python3.12/site-packages/torchvision/tv_tensors/_video.py b/lib/python3.12/site-packages/torchvision/tv_tensors/_video.py new file mode 100644 index 0000000000000000000000000000000000000000..aa923e781ef0438e88c672f4f46d884689e206af --- /dev/null +++ b/lib/python3.12/site-packages/torchvision/tv_tensors/_video.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from typing import Any, Optional, Union + +import torch + +from ._tv_tensor import TVTensor + + +class Video(TVTensor): + """:class:`torch.Tensor` subclass for videos with shape ``[..., T, C, H, W]``. + + Args: + data (tensor-like): Any data that can be turned into a tensor with :func:`torch.as_tensor`. + dtype (torch.dtype, optional): Desired data type. If omitted, will be inferred from + ``data``. + device (torch.device, optional): Desired device. If omitted and ``data`` is a + :class:`torch.Tensor`, the device is taken from it. Otherwise, the video is constructed on the CPU. + requires_grad (bool, optional): Whether autograd should record operations. If omitted and + ``data`` is a :class:`torch.Tensor`, the value is taken from it. Otherwise, defaults to ``False``. + """ + + def __new__( + cls, + data: Any, + *, + dtype: Optional[torch.dtype] = None, + device: Optional[Union[torch.device, str, int]] = None, + requires_grad: Optional[bool] = None, + ) -> Video: + tensor = cls._to_tensor(data, dtype=dtype, device=device, requires_grad=requires_grad) + if data.ndim < 4: + raise ValueError + return tensor.as_subclass(cls) + + def __repr__(self, *, tensor_contents: Any = None) -> str: # type: ignore[override] + return self._make_repr() diff --git a/lib/python3.12/site-packages/triton/__init__.py b/lib/python3.12/site-packages/triton/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..08872dae02dd7adc5397369b62472625ab45df98 --- /dev/null +++ b/lib/python3.12/site-packages/triton/__init__.py @@ -0,0 +1,73 @@ +"""isort:skip_file""" +__version__ = '3.2.0' + +# --------------------------------------- +# Note: import order is significant here. + +# submodules +from .runtime import ( + autotune, + Config, + heuristics, + JITFunction, + KernelInterface, + reinterpret, + TensorWrapper, + OutOfResources, + InterpreterError, + MockTensor, +) +from .runtime.jit import jit +from .compiler import compile, CompilationError +from .errors import TritonError + +from . import language +from . import testing +from . import tools + +__all__ = [ + "autotune", + "cdiv", + "CompilationError", + "compile", + "Config", + "heuristics", + "impl", + "InterpreterError", + "jit", + "JITFunction", + "KernelInterface", + "language", + "MockTensor", + "next_power_of_2", + "ops", + "OutOfResources", + "reinterpret", + "runtime", + "TensorWrapper", + "TritonError", + "testing", + "tools", +] + +# ------------------------------------- +# misc. utilities that don't fit well +# into any specific module +# ------------------------------------- + + +def cdiv(x: int, y: int): + return (x + y - 1) // y + + +def next_power_of_2(n: int): + """Return the smallest power of 2 greater than or equal to n""" + n -= 1 + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + n |= n >> 32 + n += 1 + return n diff --git a/lib/python3.12/site-packages/triton/_internal_testing.py b/lib/python3.12/site-packages/triton/_internal_testing.py new file mode 100644 index 0000000000000000000000000000000000000000..f8909f7c05875550f6778e4c0ff5b2084c5d14de --- /dev/null +++ b/lib/python3.12/site-packages/triton/_internal_testing.py @@ -0,0 +1,123 @@ +import os +import re +import numpy as np +import torch +import triton +import triton.language as tl +import pytest + +from numpy.random import RandomState +from typing import Optional, Union +from triton.runtime.jit import TensorWrapper, reinterpret + +int_dtypes = ['int8', 'int16', 'int32', 'int64'] +uint_dtypes = ['uint8', 'uint16', 'uint32', 'uint64'] +integral_dtypes = int_dtypes + uint_dtypes +float_dtypes = ['float16', 'float32', 'float64'] +dtypes = integral_dtypes + float_dtypes +dtypes_with_bfloat16 = dtypes + ['bfloat16'] +torch_float8_dtypes = ['float8_e4m3fn', 'float8_e5m2'] +torch_dtypes = ['bool'] + int_dtypes + ['uint8'] + float_dtypes + ['bfloat16'] + + +def is_interpreter(): + return os.environ.get('TRITON_INTERPRET', '0') == '1' + + +def get_current_target(): + if is_interpreter(): + return None + return triton.runtime.driver.active.get_current_target() + + +def is_cuda(): + target = get_current_target() + return False if target is None else target.backend == "cuda" + + +def is_hip(): + target = get_current_target() + return False if target is None else target.backend == "hip" + + +def get_arch(): + target = get_current_target() + return "" if target is None else str(target.arch) + + +def numpy_random(shape, dtype_str, rs: Optional[RandomState] = None, low=None, high=None): + """ + Override `rs` if you're calling this function twice and don't want the same + result for both calls. + """ + if isinstance(shape, int): + shape = (shape, ) + if rs is None: + rs = RandomState(seed=17) + if dtype_str in int_dtypes + uint_dtypes: + iinfo = np.iinfo(getattr(np, dtype_str)) + low = iinfo.min if low is None else max(low, iinfo.min) + high = iinfo.max if high is None else min(high, iinfo.max) + dtype = getattr(np, dtype_str) + x = rs.randint(low, high, shape, dtype=dtype) + x[x == 0] = 1 # Workaround. Never return zero so tests of division don't error out. + return x + elif dtype_str and 'float8' in dtype_str: + x = rs.randint(20, 40, shape, dtype=np.int8) + return x + elif dtype_str in float_dtypes: + return rs.normal(0, 1, shape).astype(dtype_str) + elif dtype_str == 'bfloat16': + return (rs.normal(0, 1, shape).astype('float32').view('uint32') & np.uint32(0xffff0000)).view('float32') + elif dtype_str in ['bool', 'int1', 'bool_']: + return rs.normal(0, 1, shape) > 0.0 + else: + raise RuntimeError(f'Unknown dtype {dtype_str}') + + +def to_triton(x: np.ndarray, device, dst_type=None) -> Union[TensorWrapper, torch.Tensor]: + ''' + Note: We need dst_type because the type of x can be different from dst_type. + For example: x is of type `float32`, dst_type is `bfloat16`. + If dst_type is None, we infer dst_type from x. + ''' + t = x.dtype.name + if t in uint_dtypes: + signed_type_name = t.lstrip('u') # e.g. "uint16" -> "int16" + x_signed = x.astype(getattr(np, signed_type_name)) + return reinterpret(torch.tensor(x_signed, device=device), getattr(tl, t)) + else: + if dst_type and 'float8' in dst_type: + return reinterpret(torch.tensor(x, device=device), getattr(tl, dst_type)) + if t == 'float32' and dst_type == 'bfloat16': + return torch.tensor(x, device=device).bfloat16() + return torch.tensor(x, device=device) + + +def torch_dtype_name(dtype) -> str: + if isinstance(dtype, triton.language.dtype): + return dtype.name + elif isinstance(dtype, torch.dtype): + # 'torch.int64' -> 'int64' + m = re.match(r'^torch\.(\w+)$', str(dtype)) + return m.group(1) + else: + raise TypeError(f'not a triton or torch dtype: {type(dtype)}') + + +def to_numpy(x): + if isinstance(x, TensorWrapper): + return x.base.cpu().numpy().astype(getattr(np, torch_dtype_name(x.dtype))) + elif isinstance(x, torch.Tensor): + if x.dtype is torch.bfloat16: + return x.cpu().float().numpy() + return x.cpu().numpy() + else: + raise ValueError(f"Not a triton-compatible tensor: {x}") + + +def supports_tma(): + return is_cuda() and torch.cuda.get_device_capability()[0] >= 9 + + +requires_tma = pytest.mark.skipif(not supports_tma(), reason="Requires TMA support (NVIDIA Hopper or higher)") diff --git a/lib/python3.12/site-packages/triton/backends/__init__.py b/lib/python3.12/site-packages/triton/backends/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..92ba144ba97b2113c5462eedf857089defd2b649 --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/__init__.py @@ -0,0 +1,50 @@ +import os +import importlib.util +import inspect +from dataclasses import dataclass +from .driver import DriverBase +from .compiler import BaseBackend + + +def _load_module(name, path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _find_concrete_subclasses(module, base_class): + ret = [] + for attr_name in dir(module): + attr = getattr(module, attr_name) + if isinstance(attr, type) and issubclass(attr, base_class) and not inspect.isabstract(attr): + ret.append(attr) + if len(ret) == 0: + raise RuntimeError(f"Found 0 concrete subclasses of {base_class} in {module}: {ret}") + if len(ret) > 1: + raise RuntimeError(f"Found >1 concrete subclasses of {base_class} in {module}: {ret}") + return ret[0] + + +@dataclass(frozen=True) +class Backend: + compiler: BaseBackend = None + driver: DriverBase = None + + +def _discover_backends(): + backends = dict() + root = os.path.dirname(__file__) + for name in os.listdir(root): + if not os.path.isdir(os.path.join(root, name)): + continue + if name.startswith('__'): + continue + compiler = _load_module(name, os.path.join(root, name, 'compiler.py')) + driver = _load_module(name, os.path.join(root, name, 'driver.py')) + backends[name] = Backend(_find_concrete_subclasses(compiler, BaseBackend), + _find_concrete_subclasses(driver, DriverBase)) + return backends + + +backends = _discover_backends() diff --git a/lib/python3.12/site-packages/triton/backends/compiler.py b/lib/python3.12/site-packages/triton/backends/compiler.py new file mode 100644 index 0000000000000000000000000000000000000000..cac42a6631a10391038e05fd81099c89455be63c --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/compiler.py @@ -0,0 +1,304 @@ +import os +import re +import hashlib +import subprocess + +from abc import ABCMeta, abstractmethod, abstractclassmethod +from dataclasses import dataclass +from typing import Dict, List, Tuple, Union +from types import ModuleType + +# Table that associates strings to AttrsDescriptor (sub)classes. +# In this way we can dynamically select the correct class +# constructor +_descriptor_table = {} + + +def register_descriptor(cls): + """ + Register a descriptor into the descriptor table + """ + _descriptor_table[cls.__name__] = cls + return cls + + +@register_descriptor +class AttrsDescriptor: + """ + This class handles compile-time properties for specific function parameters. + + Different backends can add more properties to the common ones. The class + contains two fields: + + `arg_properties`: a dictionary containing the different compile-time properties for different + parameters. I.e., the dictionary is a map from property names to parameter indices + { + "prop0": (0, 2, 3) + "prop1": (0, 4, 5) + } + Different backends might need different properties on those paraemters to enable + specific optimizations. The common compile time properties contained in this class + are : + - "tt.divisibility", i.e., is the given parameter divisible by 16 + - "tt.equal_to_1", i.e., is the given parameter an integer constant 1 + + `property_values`: a dictionary containing the value of the different compile-time properties, like: + { + "prop0": val0 + "prop1": val1 + } + + `constant_properties`: a set containing the properties that can be used to determine if a parameter is constant + + """ + __slots__ = ('divisibility_16', 'equal_to_1', 'arg_properties', 'property_values', 'constant_properties') + + def __init__(self, params=None, values=None): + """ + Initialize the compile-time properties + + We can initialize the AttrsDescriptor class by passing the list of params + of the function and their `values`. The function will try to apply the properties + to the values and save the parameters in the `arg_properties` list. If we don't pass + either the `params` or the `values` we should initialize the class via an alternative method + (see `from_dict` or `from_hints`) + """ + # Default initialization + self.arg_properties = {} + self.property_values = {} + self.constant_properties = set() + + self._add_common_properties(params, values) + self._add_backend_properties(params, values) + self._init_slots() + + def _add_common_properties(self, params, values): + """ Add common compile-time properties """ + self.property_values["tt.divisibility"] = 16 + self.property_values["tt.equal_to"] = 1 + self.constant_properties.add("tt.equal_to") + + if (params is None) or (values is None): + return + + # Compile properties deduction + assert (len(params) == len(values)) + + # Divisibility property + self.arg_properties["tt.divisibility"] = [ + param.num for param, arg in zip(params, values) if AttrsDescriptor.is_divisible_by_16(arg) + and not param.do_not_specialize and not param.do_not_specialize_on_alignment + ] + + # Equal to 1 property + self.arg_properties["tt.equal_to"] = [ + param.num + for param, arg in zip(params, values) + if AttrsDescriptor.is_equal_to_1(arg) and not param.do_not_specialize + ] + + def _add_backend_properties(self, params=None, values=None): + """ This method is for different subclasses to implement their own compile-time properties """ + pass + + def _init_slots(self): + """ Initialize the slots of this class """ + for name, val in self.arg_properties.items(): + setattr(self, name.removeprefix('tt.') + '_' + str(self.property_values[name]), val) + + def get_fn_attrs(self) -> Dict: + """ + Get the function attributes as a dictionary. + + The returned dictionary will look like : + { + "arg0" : [(prop_name00, val00), (prop_name01, val01), ...)]} + "arg1" : [(prop_name10, val10), (prop_name11, val11), ...)]} + } + """ + attrs = {} + for prop_name, arg_set in self.arg_properties.items(): + prop_val = self.property_values[prop_name] + for arg in arg_set: + attrs[arg] = attrs.get(arg, []) + [(prop_name, prop_val)] + return attrs + + def get_constants(self) -> Dict: + """ Return a mapping of constant parameters to their values """ + constants = {} + for prop_name in self.constant_properties: + for p in self.arg_properties.get(prop_name, []): + constants[p] = self.property_values[prop_name] + return constants + + def filter_out_constants(self): + """ Return the same object, without properties marked as constants""" + import copy + c = copy.deepcopy(self) + for prop_name in c.constant_properties: + c.arg_properties.pop(prop_name, None) + c.property_values.pop(prop_name, None) + c.constant_properties = {} + return c + + def hash(self): + values = [sorted(self.arg_properties.values())] + values += [sorted(self.property_values.values())] + values += [sorted(self.constant_properties)] + key = str(values) + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + def to_dict(self): + """ + Store the fields of this class in a serializable dictionary + """ + # We need to only store the `arg_properties` field. To initialize the + # other fields we relay on the class type. We store it as a string in + # the dictionary so that we can use it to invoke the appropriate + # (sub)class constructor in the `from_dict` method. + return {"arg_properties": self.arg_properties, "cls": type(self).__name__} + + @staticmethod + def from_dict(data): + """ + Create the object from a serializable dictionary + """ + attrs_descriptor = _descriptor_table[data["cls"]]() + for prop_name, param_ids in data["arg_properties"].items(): + attrs_descriptor.arg_properties[prop_name] = param_ids + attrs_descriptor._init_slots() + return attrs_descriptor + + @classmethod + def from_hints(cls, hints: List[Tuple[int, int]]): + """ + Create the class from a set of hints that are passed in. + + Instead of deducing the properties from a list of paramaters and values, + the user can pass in a list of `hints=[(param_index, val)]` and if `val` + matches one of the values of the properties (e.g., `prop_val[prop0]`), + then we insert `param_index` into the correct list (e.g., in + `arg_properties[prop0]`) + """ + attrs_descriptor = cls() + for prop_name, prop_val in attrs_descriptor.property_values.items(): + attrs_descriptor.arg_properties[prop_name] = [i for i, h in hints.items() if h == prop_val] + attrs_descriptor._init_slots() + return attrs_descriptor + + @staticmethod + def is_divisible_by_16(x): + """ Return if the argument is a multiple of 16""" + if hasattr(x, "data_ptr"): + return x.data_ptr() % 16 == 0 + elif isinstance(x, int): + return x % 16 == 0 + if x is None: + return True + return False + + @staticmethod + def is_equal_to_1(x): + """ Return if the argument is a constant 1""" + return True if isinstance(x, int) and not isinstance(x, bool) and x == 1 else False + + @staticmethod + def get_property_key(val, align): + if align and AttrsDescriptor.is_divisible_by_16(val): + return "D" + if AttrsDescriptor.is_equal_to_1(val): + return "1" + return "N" + + def __repr__(self): + return f"AttrsDescriptor.from_dict({self.to_dict()!r})" + + +@dataclass(frozen=True) +class GPUTarget(object): + # Target backend, e.g., cuda, hip + backend: str + # Target architecture, e.g., 90 (for cuda compute capability), gfx940 (for hip) + arch: Union[int, str] + warp_size: int + + +class BaseBackend(metaclass=ABCMeta): + + def __init__(self, target: GPUTarget) -> None: + self.target = target + assert self.supports_target(target) + + @staticmethod + def _path_to_binary(binary: str): + base_dir = os.path.join(os.path.dirname(__file__), os.pardir) + paths = [ + os.environ.get(f"TRITON_{binary.upper()}_PATH", ""), + os.path.join(base_dir, "third_party", "cuda", "bin", binary), + ] + for p in paths: + bin = p.split(" ")[0] + if os.path.exists(bin) and os.path.isfile(bin): + result = subprocess.check_output([bin, "--version"], stderr=subprocess.STDOUT) + if result is not None: + version = re.search(r".*release (\d+\.\d+).*", result.decode("utf-8"), flags=re.MULTILINE) + if version is not None: + return p, version.group(1) + raise RuntimeError(f"Cannot find {binary}") + + @abstractclassmethod + def supports_target(target: GPUTarget): + raise NotImplementedError + + @abstractmethod + def hash(self) -> str: + """Returns a unique identifier for this backend""" + raise NotImplementedError + + @abstractmethod + def parse_options(self, options: dict) -> object: + """ + Converts an `options` dictionary into an arbitrary object and returns it. + This function may contain target-specific heuristics and check the legality of the provided options + """ + raise NotImplementedError + + @abstractmethod + def add_stages(self, stages: dict, options: object) -> None: + """ + Populates `stages` dictionary with entries of the form: + ir_name [str] => Function[(src: str, metadata: dict) -> str|bytes] + The value of each entry may populate a `metadata` dictionary. + Stages will be run sequentially (in inseriton order) and can communicate using `metadata`. + All stages are expected to return a `str` object, except for the last stage which returns + a `bytes` object for execution by the launcher. + """ + raise NotImplementedError + + @abstractmethod + def load_dialects(self, context): + """ + Load additional MLIR dialects into the provided `context` + """ + raise NotImplementedError + + @abstractmethod + def get_module_map(self) -> Dict[str, ModuleType]: + """ + Return a map of interface modules to their device-specific implementations + """ + raise NotImplementedError + + def get_attrs_descriptor(self, params, args): + """ + Return an attribute descriptor: given a set of parameters and arguments + the descriptor stores a set of compile time properties that can improve code + generation. Different backends might benefit from different properties + """ + return AttrsDescriptor(params, args) + + def compute_spec_key(self, arg, align): + """ + Return the ascii key for a given argument with a given set of properties + """ + return AttrsDescriptor.get_property_key(arg, align) diff --git a/lib/python3.12/site-packages/triton/backends/driver.py b/lib/python3.12/site-packages/triton/backends/driver.py new file mode 100644 index 0000000000000000000000000000000000000000..202ae15686d416fc278e0a28b34d538af71beec3 --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/driver.py @@ -0,0 +1,48 @@ +from abc import ABCMeta, abstractmethod, abstractclassmethod +from typing import Callable, List, Protocol, Sequence + + +class Benchmarker(Protocol): + + def __call__(self, kernel_call: Callable, *, quantiles: List[float], **kwargs) -> Sequence[float]: + pass + + +class DriverBase(metaclass=ABCMeta): + + @abstractclassmethod + def is_active(self): + pass + + @abstractmethod + def get_current_target(self): + pass + + @abstractmethod + def get_benchmarker(self) -> Benchmarker: + """ + Return the benchmarking function that this backend should use by default. + """ + raise NotImplementedError + + def __init__(self) -> None: + pass + + +class GPUDriver(DriverBase): + + def __init__(self): + # TODO: support other frameworks than torch + import torch + self.get_device_capability = torch.cuda.get_device_capability + try: + from torch._C import _cuda_getCurrentRawStream + self.get_current_stream = _cuda_getCurrentRawStream + except ImportError: + self.get_current_stream = lambda idx: torch.cuda.current_stream(idx).cuda_stream + self.get_current_device = torch.cuda.current_device + self.set_current_device = torch.cuda.set_device + + # TODO: remove once TMA is cleaned up + def assemble_tensormap_to_arg(self, tensormaps_info, args): + return args diff --git a/lib/python3.12/site-packages/triton/backends/nvidia/__init__.py b/lib/python3.12/site-packages/triton/backends/nvidia/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/triton/backends/nvidia/compiler.py b/lib/python3.12/site-packages/triton/backends/nvidia/compiler.py new file mode 100644 index 0000000000000000000000000000000000000000..ac53188931624bb0490a2db349e6619fb2457616 --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/nvidia/compiler.py @@ -0,0 +1,394 @@ +from triton.backends.compiler import BaseBackend, GPUTarget +from triton._C.libtriton import ir, passes, llvm, nvidia + +from dataclasses import dataclass +import functools +from typing import Any, Dict, Tuple, Optional +from types import ModuleType +import hashlib +import re +import tempfile +import signal +import os +import subprocess +from pathlib import Path + + +def min_dot_size(target: GPUTarget): + return lambda lhsType, rhsType: (16, 32, 16) if lhsType.is_int8() else (16, 16, 16) + + +@functools.lru_cache() +def _path_to_binary(binary: str): + paths = [ + os.environ.get(f"TRITON_{binary.upper()}_PATH", ""), + os.path.join(os.path.dirname(__file__), "bin", binary), + ] + + for bin in paths: + if os.path.exists(bin) and os.path.isfile(bin): + result = subprocess.check_output([bin, "--version"], stderr=subprocess.STDOUT) + if result is not None: + version = re.search(r".*release (\d+\.\d+).*", result.decode("utf-8"), flags=re.MULTILINE) + if version is not None: + return bin, version.group(1) + raise RuntimeError(f"Cannot find {binary}") + + +@functools.lru_cache() +def get_ptxas_version(): + version = subprocess.check_output([_path_to_binary("ptxas")[0], "--version"]).decode("utf-8") + return version + + +@functools.lru_cache() +def ptx_get_version(cuda_version) -> int: + ''' + Get the highest PTX version supported by the current CUDA driver. + ''' + assert isinstance(cuda_version, str) + major, minor = map(int, cuda_version.split('.')) + if major == 12: + if minor < 6: + return 80 + minor + elif minor == 6: + return 85 + if major == 11: + return 70 + minor + if major == 10: + return 63 + minor + raise RuntimeError("Triton only support CUDA 10.0 or higher, but got CUDA version: " + cuda_version) + + +def get_ptx_version_from_options(options): + ptx_version = options.ptx_version + if ptx_version is None: + _, cuda_version = _path_to_binary("ptxas") + ptx_version = ptx_get_version(cuda_version) + return ptx_version + + +@functools.lru_cache() +def get_features(options): + ptx_version = get_ptx_version_from_options(options) + + # PTX 8.3 is the max version supported by llvm 3a83162168. + # + # To check if a newer PTX version is supported, increase this value + # and run a test. If it's not supported, LLVM will print a warning + # like "+ptx8.4 is not a recognized feature for this target". + llvm_ptx_version = min(83, ptx_version) + features = f'+ptx{llvm_ptx_version}' + return features + + +@functools.lru_cache(None) +def file_hash(path): + with open(path, "rb") as f: + return hashlib.sha256(f.read()).hexdigest() + + +@dataclass(frozen=True) +class CUDAOptions: + num_warps: int = 4 + num_ctas: int = 1 + num_stages: int = 3 + num_buffers_warp_spec: int = 0 + num_consumer_groups: int = 0 + reg_dec_producer: int = 0 + reg_inc_consumer: int = 0 + # maxnreg corresponds to the ptx parameter .maxnreg, which controls the + # maximum number of 32-bit registers used by one thread. + maxnreg: Optional[int] = None + cluster_dims: tuple = (1, 1, 1) + ptx_version: int = None + enable_fp_fusion: bool = True + supported_fp8_dtypes: Tuple[str] = ("fp8e5", "fp8e4b15") + deprecated_fp8_dtypes: Tuple[str] = () + default_dot_input_precision: str = "tf32" + allowed_dot_input_precisions: Tuple[str] = ("tf32", "tf32x3", "ieee") + max_num_imprecise_acc_default: bool = None + extern_libs: dict = None + debug: bool = False + backend_name: str = 'cuda' + sanitize_overflow: bool = True + + def __post_init__(self): + default_libdir = Path(__file__).parent / 'lib' + extern_libs = {} if self.extern_libs is None else dict(self.extern_libs) + if not extern_libs.get('libdevice', None): + extern_libs['libdevice'] = os.getenv("TRITON_LIBDEVICE_PATH", str(default_libdir / 'libdevice.10.bc')) + object.__setattr__(self, 'extern_libs', tuple(extern_libs.items())) + assert self.num_warps > 0 and (self.num_warps & (self.num_warps - 1)) == 0, \ + "num_warps must be a power of 2" + + def hash(self): + hash_dict = dict(self.__dict__) + hash_dict["extern_libs"] = tuple((k, file_hash(v)) for k, v in sorted(hash_dict["extern_libs"])) + key = "_".join([f"{name}-{val}" for name, val in sorted(hash_dict.items())]) + return hashlib.sha256(key.encode("utf-8")).hexdigest() + + +class CUDABackend(BaseBackend): + + @staticmethod + def supports_target(target: GPUTarget): + return target.backend == 'cuda' + + def __init__(self, target: GPUTarget) -> None: + super().__init__(target) + self.capability = target.arch + assert isinstance(self.capability, int) + self.binary_ext = "cubin" + + def parse_options(self, opts) -> Any: + args = {k: opts[k] for k in CUDAOptions.__dataclass_fields__.keys() if k in opts} + if "supported_fp8_dtypes" not in args: + supported_fp8_dtypes = set(CUDAOptions.supported_fp8_dtypes) + if self.capability >= 89: + supported_fp8_dtypes.add("fp8e4nv") + args["supported_fp8_dtypes"] = tuple(sorted(supported_fp8_dtypes)) + + if "deprecated_fp8_dtypes" not in args: + if self.capability >= 90: + args["deprecated_fp8_dtypes"] = ("fp8e4b15", ) + + if "enable_fp_fusion" not in args: + args["enable_fp_fusion"] = os.getenv("TRITON_DEFAULT_FP_FUSION", "1") == "1" + args["max_num_imprecise_acc_default"] = 2**30 if self.capability == 90 else 0 + return CUDAOptions(**args) + + def pack_metadata(self, metadata): + return ( + metadata.num_warps, + metadata.num_ctas, + metadata.shared, + metadata.cluster_dims[0], + metadata.cluster_dims[1], + metadata.cluster_dims[2], + ) + + def get_codegen_implementation(self): + import triton.language.extra.cuda as cuda + codegen_fns = { + "convert_custom_types": + cuda.convert_custom_float8_sm80 if self.capability >= 80 else cuda.convert_custom_float8_sm70, + "min_dot_size": min_dot_size(self.target) + } + return codegen_fns + + def get_module_map(self) -> Dict[str, ModuleType]: + from triton.language.extra.cuda import libdevice + return {"triton.language.extra.libdevice": libdevice} + + def load_dialects(self, ctx): + nvidia.load_dialects(ctx) + + @staticmethod + def make_ttir(mod, metadata, opt): + pm = ir.pass_manager(mod.context) + pm.enable_debug() + passes.common.add_inliner(pm) + passes.ttir.add_rewrite_tensor_pointer(pm) + passes.ttir.add_combine(pm) + passes.common.add_canonicalizer(pm) + passes.ttir.add_reorder_broadcast(pm) + passes.common.add_cse(pm) + passes.common.add_licm(pm) + passes.common.add_symbol_dce(pm) + passes.ttir.add_loop_unroll(pm) + pm.run(mod) + return mod + + @staticmethod + def make_ttgir(mod, metadata, opt, capability): + cluster_info = nvidia.ClusterInfo() + if opt.cluster_dims is not None: + cluster_info.clusterDimX = opt.cluster_dims[0] + cluster_info.clusterDimY = opt.cluster_dims[1] + cluster_info.clusterDimZ = opt.cluster_dims[2] + # Set up Diagnostic + if os.environ.get("MLIR_ENABLE_REMARK", "0") == "1": + srcMgr = llvm.source_mgr() + diag = ir.source_mgr_diag(srcMgr, mod.context) + mod.context.printOpOnDiagnostic(True) + # TTIR -> TTGIR + pm = ir.pass_manager(mod.context) + pm.enable_debug() + passes.ttir.add_convert_to_ttgpuir(pm, f"cuda:{capability}", opt.num_warps, 32, opt.num_ctas) + # optimize TTGIR + passes.ttgpuir.add_coalesce(pm) + if capability // 10 >= 8: + passes.ttgpuir.add_f32_dot_tc(pm) + # TODO(Qingyi): Move PlanCTAPass to the front of CoalescePass + nvidia.passes.ttnvgpuir.add_plan_cta(pm, cluster_info) + passes.ttgpuir.add_remove_layout_conversions(pm) + passes.ttgpuir.add_optimize_thread_locality(pm) + passes.ttgpuir.add_accelerate_matmul(pm) + passes.ttgpuir.add_remove_layout_conversions(pm) + passes.ttgpuir.add_optimize_dot_operands(pm, capability >= 80) + passes.common.add_cse(pm) + if capability // 10 >= 8: + passes.ttgpuir.add_optimize_accumulator_init(pm) + passes.ttgpuir.add_combine_tensor_select_and_if(pm) + passes.ttgpuir.add_ws_task_partition(pm, opt.num_consumer_groups) + passes.ttgpuir.add_taskid_propagate(pm, opt.num_consumer_groups) + passes.ttgpuir.add_ws_data_partition(pm, opt.num_consumer_groups) + passes.ttgpuir.add_ws_code_partition(pm, opt.num_buffers_warp_spec, opt.num_consumer_groups, + opt.reg_dec_producer, opt.reg_inc_consumer) + passes.ttgpuir.add_pipeline(pm, opt.num_stages) + passes.ttgpuir.add_ws_lowering(pm, opt.num_consumer_groups) + passes.ttgpuir.add_prefetch(pm) + passes.ttgpuir.add_optimize_dot_operands(pm, capability >= 80) + passes.ttgpuir.add_remove_layout_conversions(pm) + passes.ttgpuir.add_reduce_data_duplication(pm) + passes.ttgpuir.add_reorder_instructions(pm) + passes.common.add_cse(pm) + passes.common.add_symbol_dce(pm) + if capability // 10 >= 9: + nvidia.passes.ttnvgpuir.add_fence_insertion(pm) + nvidia.passes.ttnvgpuir.add_tma_lowering(pm) + passes.common.add_canonicalizer(pm) + pm.run(mod) + metadata["cluster_dims"] = (cluster_info.clusterDimX, cluster_info.clusterDimY, cluster_info.clusterDimZ) + return mod + + @staticmethod + def make_llir(src, metadata, options, capability): + ptx_version = get_ptx_version_from_options(options) + + # warp-specialization mutates num_warps + num_warp_groups = src.get_int_attr("triton_gpu.num-warp-groups-per-cta") + if num_warp_groups is not None: + metadata["num_warps"] *= num_warp_groups + mod = src + # TritonGPU -> LLVM-IR (MLIR) + pm = ir.pass_manager(mod.context) + pm.enable_debug() + # Set up Diagnostic + if os.environ.get("MLIR_ENABLE_REMARK", "0") == "1": + srcMgr = llvm.source_mgr() + diag = ir.source_mgr_diag(srcMgr, mod.context) + mod.context.printOpOnDiagnostic(True) + nvidia.passes.ttgpuir.add_decompose_unsupported_conversions(pm) + passes.ttgpuir.add_combine_tensor_select_and_if(pm) + passes.convert.add_scf_to_cf(pm) + passes.convert.add_index_to_llvmir(pm) + passes.ttgpuir.add_allocate_shared_memory(pm) + nvidia.passes.ttgpuir.add_to_llvmir(pm, capability, ptx_version) + nvidia.passes.ttnvgpuir.add_nvgpu_to_llvm(pm) + passes.convert.add_arith_to_llvmir(pm) + passes.common.add_canonicalizer(pm) + passes.common.add_cse(pm) + passes.common.add_symbol_dce(pm) + if os.environ.get("TRITON_DISABLE_LINE_INFO", "0") == "0": + passes.llvmir.add_di_scope(pm) + pm.run(mod) + # LLVM-IR (MLIR) -> LLVM-IR (LLVM) + llvm.init_targets() + context = llvm.context() + + llvm_mod = llvm.to_module(mod, context) + proc = 'sm_90a' if capability == 90 else f'sm_{capability}' + features = get_features(options) + triple = 'nvptx64-nvidia-cuda' + llvm.attach_datalayout(llvm_mod, triple, proc, features) + nvidia.set_nvvm_reflect_ftz(llvm_mod) + + # Set maxnreg on all kernels, if it was provided. + if options.maxnreg is not None: + for k in llvm_mod.get_functions(): + if not k.is_declaration() and k.is_external_linkage(): + k.set_nvvm_maxnreg(options.maxnreg) + + if options.extern_libs: + paths = [path for (name, path) in options.extern_libs] + llvm.link_extern_libs(llvm_mod, paths) + + llvm.optimize_module(llvm_mod, llvm.OPTIMIZE_O3) + + # Get some metadata + metadata["shared"] = src.get_int_attr("triton_gpu.shared") + ret = str(llvm_mod) + del llvm_mod + del context + return ret + + @staticmethod + def make_ptx(src, metadata, opt, capability): + ptx_version = get_ptx_version_from_options(opt) + + triple = 'nvptx64-nvidia-cuda' + proc = 'sm_90a' if capability == 90 else f'sm_{capability}' + features = get_features(opt) + ret = llvm.translate_to_asm(src, triple, proc, features, ['nvptx-short-ptr'], opt.enable_fp_fusion, False) + # Find kernel names (there should only be one) + names = re.findall(r".visible .entry ([a-zA-Z_][a-zA-Z0-9_]*)", ret) + assert len(names) == 1 + metadata["name"] = names[0] + # post-process + ptx_version = f'{ptx_version//10}.{ptx_version%10}' + ret = re.sub(r'\.version \d+\.\d+', f'.version {ptx_version}', ret, flags=re.MULTILINE) + # Remove the debug flag that prevents ptxas from optimizing the code + ret = re.sub(r",\s*debug|debug,\s*", "", ret) + if os.environ.get("NVPTX_ENABLE_DUMP", "0") == "1": + print("// -----// NVPTX Dump //----- //") + print(ret) + return ret + + @staticmethod + def make_cubin(src, metadata, opt, capability): + ptxas, _ = _path_to_binary("ptxas") + with tempfile.NamedTemporaryFile(delete=False, mode='w', suffix='.ptx') as fsrc, \ + tempfile.NamedTemporaryFile(delete=False, mode='r', suffix='.log') as flog: + fsrc.write(src) + fsrc.flush() + fbin = fsrc.name + '.o' + + line_info = [] if os.environ.get('TRITON_DISABLE_LINE_INFO') else ['-lineinfo'] + fmad = [] if opt.enable_fp_fusion else ['--fmad=false'] + suffix = 'a' if capability == 90 else '' + opt_level = ['--opt-level', '0'] if os.environ.get("DISABLE_PTXAS_OPT", "0") == "1" else [] + ptxas_cmd = [ + ptxas, *line_info, *fmad, '-v', *opt_level, f'--gpu-name=sm_{capability}{suffix}', fsrc.name, '-o', fbin + ] + try: + subprocess.run(ptxas_cmd, check=True, close_fds=False, stderr=flog) + if os.path.exists(fsrc.name): + os.remove(fsrc.name) + if os.path.exists(flog.name): + os.remove(flog.name) + except subprocess.CalledProcessError as e: + with open(flog.name) as log_file: + log = log_file.read() + if os.path.exists(flog.name): + os.remove(flog.name) + + if e.returncode == 255: + error = 'Internal Triton PTX codegen error' + elif e.returncode == 128 + signal.SIGSEGV: + error = '`ptxas` raised SIGSEGV' + else: + error = f'`ptxas` failed with error code {e.returncode}' + + raise RuntimeError(f'{error}\n' + f'`ptxas` stderr:\n{log}\n' + f'Repro command: {" ".join(ptxas_cmd)}\n') + + with open(fbin, 'rb') as f: + cubin = f.read() + if os.path.exists(fbin): + os.remove(fbin) + return cubin + + def add_stages(self, stages, options): + stages["ttir"] = lambda src, metadata: self.make_ttir(src, metadata, options) + stages["ttgir"] = lambda src, metadata: self.make_ttgir(src, metadata, options, self.capability) + stages["llir"] = lambda src, metadata: self.make_llir(src, metadata, options, self.capability) + stages["ptx"] = lambda src, metadata: self.make_ptx(src, metadata, options, self.capability) + stages["cubin"] = lambda src, metadata: self.make_cubin(src, metadata, options, self.capability) + + @functools.lru_cache() + def hash(self): + version = get_ptxas_version() + return f'{version}-{self.capability}' diff --git a/lib/python3.12/site-packages/triton/backends/nvidia/driver.c b/lib/python3.12/site-packages/triton/backends/nvidia/driver.c new file mode 100644 index 0000000000000000000000000000000000000000..bb0d868881205c52d12779a5a452562333e65135 --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/nvidia/driver.c @@ -0,0 +1,422 @@ +#include "cuda.h" +#include +#include +#define PY_SSIZE_T_CLEAN +#include +#include + +// Raises a Python exception and returns false if code is not CUDA_SUCCESS. +static bool gpuAssert(CUresult code, const char *file, int line) { + if (code == CUDA_SUCCESS) + return true; + + const char *prefix = "Triton Error [CUDA]: "; + const char *str; + cuGetErrorString(code, &str); + char err[1024] = {0}; + strcat(err, prefix); + strcat(err, str); + PyGILState_STATE gil_state; + gil_state = PyGILState_Ensure(); + PyErr_SetString(PyExc_RuntimeError, err); + PyGILState_Release(gil_state); + return false; +} + +// To be used only *outside* a Py_{BEGIN,END}_ALLOW_THREADS block. +#define CUDA_CHECK_AND_RETURN_NULL(ans) \ + do { \ + if (!gpuAssert((ans), __FILE__, __LINE__)) \ + return NULL; \ + } while (0) + +// To be used inside a Py_{BEGIN,END}_ALLOW_THREADS block. +#define CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(ans) \ + do { \ + if (!gpuAssert((ans), __FILE__, __LINE__)) { \ + PyEval_RestoreThread(_save); \ + return NULL; \ + } \ + } while (0) + +// Used to check if functions exist in old CUDA driver versions. +#define INITIALIZE_FUNCTION_POINTER_IF_NULL(funcPointer, initializerFunction) \ + do { \ + if ((funcPointer) == NULL) { \ + (funcPointer) = (initializerFunction)(); \ + if ((funcPointer) == NULL) { \ + return NULL; \ + } \ + } \ + } while (0) + +static PyObject *getDeviceProperties(PyObject *self, PyObject *args) { + int device_id; + if (!PyArg_ParseTuple(args, "i", &device_id)) + return NULL; + // Get device handle + CUdevice device; + cuDeviceGet(&device, device_id); + + // create a struct to hold device properties + int max_shared_mem; + int max_num_regs; + int multiprocessor_count; + int warp_size; + int sm_clock_rate; + int mem_clock_rate; + int mem_bus_width; + CUDA_CHECK_AND_RETURN_NULL(cuDeviceGetAttribute( + &max_shared_mem, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, + device)); + CUDA_CHECK_AND_RETURN_NULL(cuDeviceGetAttribute( + &max_num_regs, CU_DEVICE_ATTRIBUTE_MAX_REGISTERS_PER_BLOCK, device)); + CUDA_CHECK_AND_RETURN_NULL(cuDeviceGetAttribute( + &multiprocessor_count, CU_DEVICE_ATTRIBUTE_MULTIPROCESSOR_COUNT, device)); + CUDA_CHECK_AND_RETURN_NULL( + cuDeviceGetAttribute(&warp_size, CU_DEVICE_ATTRIBUTE_WARP_SIZE, device)); + CUDA_CHECK_AND_RETURN_NULL(cuDeviceGetAttribute( + &sm_clock_rate, CU_DEVICE_ATTRIBUTE_CLOCK_RATE, device)); + CUDA_CHECK_AND_RETURN_NULL(cuDeviceGetAttribute( + &mem_clock_rate, CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE, device)); + CUDA_CHECK_AND_RETURN_NULL(cuDeviceGetAttribute( + &mem_bus_width, CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH, device)); + + return Py_BuildValue("{s:i, s:i, s:i, s:i, s:i, s:i, s:i}", "max_shared_mem", + max_shared_mem, "max_num_regs", max_num_regs, + "multiprocessor_count", multiprocessor_count, "warpSize", + warp_size, "sm_clock_rate", sm_clock_rate, + "mem_clock_rate", mem_clock_rate, "mem_bus_width", + mem_bus_width); +} + +static PyObject *loadBinary(PyObject *self, PyObject *args) { + const char *name; + const char *data; + Py_ssize_t data_size; + int shared; + int device; + if (!PyArg_ParseTuple(args, "ss#ii", &name, &data, &data_size, &shared, + &device)) { + return NULL; + } + CUfunction fun; + CUmodule mod; + int32_t n_regs = 0; + int32_t n_spills = 0; + // create driver handles + CUcontext pctx = 0; + + Py_BEGIN_ALLOW_THREADS; + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuCtxGetCurrent(&pctx)); + if (!pctx) { + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuDevicePrimaryCtxRetain(&pctx, device)); + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuCtxSetCurrent(pctx)); + } + + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuModuleLoadData(&mod, data)); + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuModuleGetFunction(&fun, mod, name)); + // get allocated registers and spilled registers from the function + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuFuncGetAttribute(&n_regs, CU_FUNC_ATTRIBUTE_NUM_REGS, fun)); + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuFuncGetAttribute(&n_spills, CU_FUNC_ATTRIBUTE_LOCAL_SIZE_BYTES, fun)); + n_spills /= 4; + // set dynamic shared memory if necessary + int shared_optin; + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuDeviceGetAttribute( + &shared_optin, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, + device)); + if (shared > 49152 && shared_optin > 49152) { + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuFuncSetCacheConfig(fun, CU_FUNC_CACHE_PREFER_SHARED)); + int shared_total, shared_static; + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuDeviceGetAttribute( + &shared_total, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_MULTIPROCESSOR, + device)); + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuFuncGetAttribute( + &shared_static, CU_FUNC_ATTRIBUTE_SHARED_SIZE_BYTES, fun)); + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuFuncSetAttribute(fun, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, + shared_optin - shared_static)); + } + Py_END_ALLOW_THREADS; + + if (PyErr_Occurred()) { + return NULL; + } + return Py_BuildValue("(KKii)", (uint64_t)mod, (uint64_t)fun, n_regs, + n_spills); +} + +typedef CUresult (*cuOccupancyMaxActiveClusters_t)( + int *numClusters, CUfunction func, const CUlaunchConfig *config); + +typedef CUresult (*cuTensorMapEncodeTiled_t)( + CUtensorMap *tensorMap, CUtensorMapDataType tensorDataType, + cuuint32_t tensorRank, void *globalAddress, const cuuint64_t *globalDim, + const cuuint64_t *globalStrides, const cuuint32_t *boxDim, + const cuuint32_t *elementStrides, CUtensorMapInterleave interleave, + CUtensorMapSwizzle swizzle, CUtensorMapL2promotion l2Promotion, + CUtensorMapFloatOOBfill oobFill); + +#define defineGetFunctionHandle(name, symbolName) \ + static symbolName##_t name() { \ + /* Open the shared library */ \ + void *libHandle = dlopen("libcuda.so.1", RTLD_LAZY); \ + if (!libHandle) { \ + PyErr_SetString(PyExc_RuntimeError, "Failed to open libcuda.so.1"); \ + return NULL; \ + } \ + /* Clear any existing error */ \ + dlerror(); \ + symbolName##_t funcHandle = (symbolName##_t)dlsym(libHandle, #symbolName); \ + /* Check for errors */ \ + const char *err = dlerror(); \ + if (err) { \ + PyErr_SetString(PyExc_RuntimeError, \ + "Failed to retrieve " #symbolName " from libcuda.so.1"); \ + dlclose(libHandle); \ + return NULL; \ + } \ + return funcHandle; \ + } + +defineGetFunctionHandle(getCuOccupancyMaxActiveClustersHandle, + cuOccupancyMaxActiveClusters); + +defineGetFunctionHandle(getCuTensorMapEncodeTiledHandle, + cuTensorMapEncodeTiled); + +static PyObject *occupancyMaxActiveClusters(PyObject *self, PyObject *args) { + int clusterDimX = -1, clusterDimY = -1, clusterDimZ = -1, + maxActiveClusters = -1; + int shared = 0; + CUfunction func; + + if (!PyArg_ParseTuple(args, "Kiiii", &func, &shared, &clusterDimX, + &clusterDimY, &clusterDimZ)) { + return NULL; + } + + // Let each SM have one block + int maxActiveBlocks = 1; + Py_BEGIN_ALLOW_THREADS; + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuFuncSetAttribute( + func, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, shared)); + Py_END_ALLOW_THREADS; + + CUlaunchAttribute launchAttr[1]; + launchAttr[0].id = CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION; + launchAttr[0].value.clusterDim.x = clusterDimX; + launchAttr[0].value.clusterDim.y = clusterDimY; + launchAttr[0].value.clusterDim.z = clusterDimZ; + CUlaunchConfig config; + config.gridDimX = clusterDimX; + config.gridDimY = maxActiveBlocks * clusterDimY; + config.gridDimZ = clusterDimZ; + config.blockDimX = 128; + config.blockDimY = 1; + config.blockDimZ = 1; + config.sharedMemBytes = shared; + config.hStream = 0; + config.numAttrs = 1; + config.attrs = launchAttr; + + static cuOccupancyMaxActiveClusters_t cuOccupancyMaxActiveClusters = NULL; + INITIALIZE_FUNCTION_POINTER_IF_NULL(cuOccupancyMaxActiveClusters, + getCuOccupancyMaxActiveClustersHandle); + + Py_BEGIN_ALLOW_THREADS; + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuFuncSetAttribute( + func, CU_FUNC_ATTRIBUTE_NON_PORTABLE_CLUSTER_SIZE_ALLOWED, 1)); + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuOccupancyMaxActiveClusters(&maxActiveClusters, func, &config)); + Py_END_ALLOW_THREADS; + return PyLong_FromLong(maxActiveClusters); +} + +static PyObject *setPrintfFifoSize(PyObject *self, PyObject *args) { + long size; + if (!PyArg_ParseTuple(args, "l", &size)) { + return NULL; + } + if (size < 0) { + PyErr_SetString(PyExc_ValueError, "fifo size must be non-negative"); + return NULL; + } + + Py_BEGIN_ALLOW_THREADS; + + // Ensure we have an active context. + CUcontext ctx = NULL; + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuCtxGetCurrent(&ctx)); + if (!ctx) { + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuDevicePrimaryCtxRetain(&ctx, /*device=*/0)); + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS(cuCtxSetCurrent(ctx)); + } + + // We can't set the fifo size after running a kernel that calls printf. This + // is true even if the set() call is a nop and the new size is the same as the + // old size. + // + // This is unfriendly, so check if the old size matches the new size, and skip + // the set() call if so. + size_t oldSize = 0; + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuCtxGetLimit(&oldSize, CU_LIMIT_PRINTF_FIFO_SIZE)); + if (oldSize != size) { + CUDA_CHECK_AND_RETURN_NULL_ALLOW_THREADS( + cuCtxSetLimit(CU_LIMIT_PRINTF_FIFO_SIZE, size)); + } + + Py_END_ALLOW_THREADS; + Py_INCREF(Py_None); + return Py_None; +} + +// Simple helper to experiment creating TMA descriptors on the host. +// This is a useful to test TMA operations independently. +static PyObject *fill1DTMADescriptor(PyObject *self, PyObject *args) { + unsigned long long global_address; + uint64_t dim; + uint32_t tensorDim; + int elementSize; + unsigned long long desc_address; + if (!PyArg_ParseTuple(args, "KKiiK", &global_address, &dim, &tensorDim, + &elementSize, &desc_address)) { + return NULL; + } + uint64_t dims[1] = {dim}; + uint64_t globalStrides[1] = {dim * elementSize}; + uint32_t boxDim[1] = {tensorDim}; + uint32_t elementStrides[1] = {1}; + CUtensorMapDataType type; + switch (elementSize) { + case 1: + type = CU_TENSOR_MAP_DATA_TYPE_UINT8; + break; + case 2: + type = CU_TENSOR_MAP_DATA_TYPE_UINT16; + break; + case 4: + type = CU_TENSOR_MAP_DATA_TYPE_UINT32; + break; + default: + PyErr_SetString(PyExc_ValueError, "elementSize must be 1, 2, or 4"); + return NULL; + } + assert((elementSize * tensorDim) >= 32 && "block size too small."); + int rank = 1; + static cuTensorMapEncodeTiled_t cuTensorMapEncodeTiled = NULL; + INITIALIZE_FUNCTION_POINTER_IF_NULL(cuTensorMapEncodeTiled, + getCuTensorMapEncodeTiledHandle); + CUDA_CHECK_AND_RETURN_NULL(cuTensorMapEncodeTiled( + (CUtensorMap *)desc_address, type, rank, (void *)global_address, dims, + globalStrides, boxDim, elementStrides, CU_TENSOR_MAP_INTERLEAVE_NONE, + CU_TENSOR_MAP_SWIZZLE_NONE, CU_TENSOR_MAP_L2_PROMOTION_NONE, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE)); + Py_INCREF(Py_None); + return Py_None; +} + +// Simple helper to experiment creating TMA descriptors on the host. +// This is a useful to test TMA operations independently. +static PyObject *fill2DTMADescriptor(PyObject *self, PyObject *args) { + unsigned long long global_address; + uint64_t dims[2]; + uint32_t tensorDims[2]; + int elementSize; + unsigned long long desc_address; + if (!PyArg_ParseTuple(args, "KKKiiiK", &global_address, &dims[1], &dims[0], + &tensorDims[1], &tensorDims[0], &elementSize, + &desc_address)) { + return NULL; + } + uint64_t globalStrides[2] = {dims[0] * elementSize, + dims[0] * dims[1] * elementSize}; + uint32_t elementStrides[2] = {1, 1}; + CUtensorMapDataType type; + switch (elementSize) { + case 1: + type = CU_TENSOR_MAP_DATA_TYPE_UINT8; + break; + case 2: + type = CU_TENSOR_MAP_DATA_TYPE_UINT16; + break; + case 4: + type = CU_TENSOR_MAP_DATA_TYPE_UINT32; + break; + default: + PyErr_SetString(PyExc_ValueError, "elementSize must be 1, 2, or 4"); + } + int rank = 2; + // Swizzling should be picked in codegen but since we need to set it on the + // descriptor we rely on a convention between this function and codegen. + CUtensorMapSwizzle swizzle = CU_TENSOR_MAP_SWIZZLE_128B; + uint32_t contigDimSizeInByte = elementSize * tensorDims[0]; + if (contigDimSizeInByte >= 128) { + swizzle = CU_TENSOR_MAP_SWIZZLE_128B; + } else if (contigDimSizeInByte >= 64) { + swizzle = CU_TENSOR_MAP_SWIZZLE_64B; + } else if (contigDimSizeInByte >= 32) { + swizzle = CU_TENSOR_MAP_SWIZZLE_32B; + } else { + assert(false && "block size too small."); + } + // The bounding box inner dimension must be less than or equal to the swizzle + // size. + // https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__TENSOR__MEMORY.html#group__CUDA__TENSOR__MEMORY_1ga7c7d2aaac9e49294304e755e6f341d7 + // We clamp the block size and the codegen will emit multiple copy operations. + if (contigDimSizeInByte > 128) { + tensorDims[0] = 128 / elementSize; + } + static cuTensorMapEncodeTiled_t cuTensorMapEncodeTiled = NULL; + INITIALIZE_FUNCTION_POINTER_IF_NULL(cuTensorMapEncodeTiled, + getCuTensorMapEncodeTiledHandle); + CUDA_CHECK_AND_RETURN_NULL(cuTensorMapEncodeTiled( + (CUtensorMap *)desc_address, type, rank, (void *)global_address, dims, + globalStrides, tensorDims, elementStrides, CU_TENSOR_MAP_INTERLEAVE_NONE, + swizzle, CU_TENSOR_MAP_L2_PROMOTION_L2_128B, + CU_TENSOR_MAP_FLOAT_OOB_FILL_NONE)); + Py_INCREF(Py_None); + return Py_None; +} + +static PyMethodDef ModuleMethods[] = { + {"load_binary", loadBinary, METH_VARARGS, + "Load provided cubin into CUDA driver"}, + {"get_device_properties", getDeviceProperties, METH_VARARGS, + "Get the properties for a given device"}, + {"cuOccupancyMaxActiveClusters", occupancyMaxActiveClusters, METH_VARARGS, + "Python interface for cuOccupancyMaxActiveClusters function"}, + {"set_printf_fifo_size", setPrintfFifoSize, METH_VARARGS, + "Python interface for cuCtxSetLimit(CU_LIMIT_PRINTF_FIFO_SIZE, x), which " + "controls how many bytes can be streamed from kernels before data starts " + "being dropped. This inherits all the limitations of this call; in " + "particular it's an error to change this value after launching any kernel " + "that calls printf()."}, + {"fill_1d_tma_descriptor", fill1DTMADescriptor, METH_VARARGS, "doc"}, + {"fill_2d_tma_descriptor", fill2DTMADescriptor, METH_VARARGS, "doc"}, + + {NULL, NULL, 0, NULL} // sentinel +}; + +static struct PyModuleDef ModuleDef = {PyModuleDef_HEAD_INIT, "cuda_utils", + NULL, // documentation + -1, // size + ModuleMethods}; + +PyMODINIT_FUNC PyInit_cuda_utils(void) { + PyObject *m = PyModule_Create(&ModuleDef); + if (m == NULL) { + return NULL; + } + + PyModule_AddFunctions(m, ModuleMethods); + + return m; +} diff --git a/lib/python3.12/site-packages/triton/backends/nvidia/driver.py b/lib/python3.12/site-packages/triton/backends/nvidia/driver.py new file mode 100644 index 0000000000000000000000000000000000000000..0a62c378c4232322cfcf886389771f087a57eb52 --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/nvidia/driver.py @@ -0,0 +1,481 @@ +import functools +import os +import hashlib +import subprocess +import tempfile +from pathlib import Path +from triton.runtime.build import _build +from triton.runtime.cache import get_cache_manager +from triton.backends.compiler import GPUTarget +from triton.backends.driver import GPUDriver + +dirname = os.path.dirname(os.path.realpath(__file__)) +include_dir = [os.path.join(dirname, "include")] +libdevice_dir = os.path.join(dirname, "lib") +libraries = ['cuda'] + + +@functools.lru_cache() +def libcuda_dirs(): + env_libcuda_path = os.getenv("TRITON_LIBCUDA_PATH") + if env_libcuda_path: + return [env_libcuda_path] + + libs = subprocess.check_output(["/sbin/ldconfig", "-p"]).decode() + # each line looks like the following: + # libcuda.so.1 (libc6,x86-64) => /lib/x86_64-linux-gnu/libcuda.so.1 + locs = [line.split()[-1] for line in libs.splitlines() if "libcuda.so.1" in line] + dirs = [os.path.dirname(loc) for loc in locs] + env_ld_library_path = os.getenv("LD_LIBRARY_PATH") + if env_ld_library_path and not dirs: + dirs = [dir for dir in env_ld_library_path.split(":") if os.path.exists(os.path.join(dir, "libcuda.so.1"))] + msg = 'libcuda.so cannot found!\n' + if locs: + msg += 'Possible files are located at %s.' % str(locs) + msg += 'Please create a symlink of libcuda.so to any of the files.' + else: + msg += 'Please make sure GPU is set up and then run "/sbin/ldconfig"' + msg += ' (requires sudo) to refresh the linker cache.' + assert any(os.path.exists(os.path.join(path, 'libcuda.so.1')) for path in dirs), msg + return dirs + + +@functools.lru_cache() +def library_dirs(): + return [libdevice_dir, *libcuda_dirs()] + + +def compile_module_from_src(src, name): + key = hashlib.sha256(src.encode("utf-8")).hexdigest() + cache = get_cache_manager(key) + cache_path = cache.get_file(f"{name}.so") + if cache_path is None: + with tempfile.TemporaryDirectory() as tmpdir: + src_path = os.path.join(tmpdir, "main.c") + with open(src_path, "w") as f: + f.write(src) + so = _build(name, src_path, tmpdir, library_dirs(), include_dir, libraries) + with open(so, "rb") as f: + cache_path = cache.put(f.read(), f"{name}.so", binary=True) + import importlib.util + spec = importlib.util.spec_from_file_location(name, cache_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + return mod + + +# ------------------------ +# Utils +# ------------------------ + + +class CudaUtils(object): + + def __new__(cls): + if not hasattr(cls, "instance"): + cls.instance = super(CudaUtils, cls).__new__(cls) + return cls.instance + + def __init__(self): + mod = compile_module_from_src(Path(os.path.join(dirname, "driver.c")).read_text(), "cuda_utils") + self.load_binary = mod.load_binary + self.get_device_properties = mod.get_device_properties + self.cuOccupancyMaxActiveClusters = mod.cuOccupancyMaxActiveClusters + self.set_printf_fifo_size = mod.set_printf_fifo_size + self.fill_1d_tma_descriptor = mod.fill_1d_tma_descriptor + self.fill_2d_tma_descriptor = mod.fill_2d_tma_descriptor + + +# ------------------------ +# Launcher +# ------------------------ + + +def ty_to_cpp(ty): + if ty[0] == '*': + return "CUdeviceptr" + return { + "i1": "int32_t", + "i8": "int8_t", + "i16": "int16_t", + "i32": "int32_t", + "i64": "int64_t", + "u1": "uint32_t", + "u8": "uint8_t", + "u16": "uint16_t", + "u32": "uint32_t", + "u64": "uint64_t", + "fp16": "float", + "bf16": "float", + "fp32": "float", + "f32": "float", + "fp64": "double", + "nvTmaDesc": "CUtensorMap", + }[ty] + + +def make_launcher(constants, signature, ids): + # Record the end of regular arguments; + # subsequent arguments are architecture-specific descriptors, such as tensor descriptors for CUDA. + arg_decls = ', '.join(f"{ty_to_cpp(ty)} arg{i}" for i, ty in signature.items()) + + def _extracted_type(ty): + if ty[0] == '*': + return "PyObject*" + if ty == "nvTmaDesc": + return "PyObject*" + + return ty_to_cpp(ty) + + def format_of(ty): + return { + "PyObject*": "O", + "float": "f", + "double": "d", + "long": "l", + "int8_t": "b", + "int16_t": "h", + "int32_t": "i", + "int64_t": "l", + "uint8_t": "B", + "uint16_t": "H", + "uint32_t": "I", + "uint64_t": "K", + }[ty] + + args_format = ''.join([format_of(_extracted_type(ty)) for ty in signature.values()]) + format = "iiiKKOOOO" + args_format + args_list = ', ' + ', '.join(f"&_arg{i}" for i, ty in signature.items()) if len(signature) > 0 else '' + + internal_args_list = [] + for i, ty in signature.items(): + if ty[0] == "*": + internal_args_list.append(f"ptr_info{i}.dev_ptr") + elif ty == "nvTmaDesc": + # Note: we have to dereference the pointer + internal_args_list.append(f"*tma_ptr{i}") + else: + internal_args_list.append(f"_arg{i}") + + # generate glue code + params = [i for i in signature.keys() if i not in constants] + src = f""" +#include \"cuda.h\" +#include +#include +#include + +static inline void gpuAssert(CUresult code, const char *file, int line) +{{ + if (code != CUDA_SUCCESS) + {{ + const char* prefix = "Triton Error [CUDA]: "; + const char* str; + cuGetErrorString(code, &str); + char err[1024] = {{0}}; + strcat(err, prefix); + strcat(err, str); + PyGILState_STATE gil_state; + gil_state = PyGILState_Ensure(); + PyErr_SetString(PyExc_RuntimeError, err); + PyGILState_Release(gil_state); + }} +}} + +#define CUDA_CHECK(ans) {{ gpuAssert((ans), __FILE__, __LINE__); }} + +typedef CUresult (*cuLaunchKernelEx_t)(const CUlaunchConfig* config, CUfunction f, void** kernelParams, void** extra); + +static cuLaunchKernelEx_t getLaunchKernelExHandle() {{ + // Open the shared library + void* handle = dlopen("libcuda.so.1", RTLD_LAZY); + if (!handle) {{ + PyErr_SetString(PyExc_RuntimeError, "Failed to open libcuda.so.1"); + return NULL; + }} + // Clear any existing error + dlerror(); + cuLaunchKernelEx_t cuLaunchKernelExHandle = (cuLaunchKernelEx_t)dlsym(handle, "cuLaunchKernelEx"); + // Check for errors + const char *dlsym_error = dlerror(); + if (dlsym_error) {{ + PyErr_SetString(PyExc_RuntimeError, "Failed to retrieve cuLaunchKernelEx from libcuda.so.1"); + return NULL; + }} + return cuLaunchKernelExHandle; +}} + +static void _launch(int gridX, int gridY, int gridZ, int num_warps, int num_ctas, int clusterDimX, int clusterDimY, int clusterDimZ, int shared_memory, CUstream stream, CUfunction function{', ' + arg_decls if len(arg_decls) > 0 else ''}) {{ + void *params[] = {{ {', '.join(f"&arg{i}" for i in params)} }}; + if (gridX*gridY*gridZ > 0) {{ + if (num_ctas == 1) {{ + CUDA_CHECK(cuLaunchKernel(function, gridX, gridY, gridZ, 32*num_warps, 1, 1, shared_memory, stream, params, 0)); + }} else {{ + CUlaunchAttribute launchAttr[2]; + launchAttr[0].id = CU_LAUNCH_ATTRIBUTE_CLUSTER_DIMENSION; + launchAttr[0].value.clusterDim.x = clusterDimX; + launchAttr[0].value.clusterDim.y = clusterDimY; + launchAttr[0].value.clusterDim.z = clusterDimZ; + launchAttr[1].id = CU_LAUNCH_ATTRIBUTE_CLUSTER_SCHEDULING_POLICY_PREFERENCE; + launchAttr[1].value.clusterSchedulingPolicyPreference = CU_CLUSTER_SCHEDULING_POLICY_SPREAD; + CUlaunchConfig config; + config.gridDimX = gridX * clusterDimX; + config.gridDimY = gridY * clusterDimY; + config.gridDimZ = gridZ * clusterDimZ; + config.blockDimX = 32 * num_warps; + config.blockDimY = 1; + config.blockDimZ = 1; + config.sharedMemBytes = shared_memory; + config.hStream = stream; + config.attrs = launchAttr; + config.numAttrs = 2; + static cuLaunchKernelEx_t cuLaunchKernelExHandle = NULL; + if (cuLaunchKernelExHandle == NULL) {{ + cuLaunchKernelExHandle = getLaunchKernelExHandle(); + }} + CUDA_CHECK(cuLaunchKernelExHandle(&config, function, params, 0)); + }} + }} +}} + +typedef struct _DevicePtrInfo {{ + CUdeviceptr dev_ptr; + bool valid; +}} DevicePtrInfo; + +static inline DevicePtrInfo getPointer(PyObject *obj, int idx) {{ + DevicePtrInfo ptr_info; + ptr_info.dev_ptr = 0; + ptr_info.valid = true; + if (PyLong_Check(obj)) {{ + ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(obj); + return ptr_info; + }} + if (obj == Py_None) {{ + // valid nullptr + return ptr_info; + }} + PyObject *ptr = PyObject_GetAttrString(obj, "data_ptr"); + if(ptr){{ + PyObject *empty_tuple = PyTuple_New(0); + PyObject *ret = PyObject_Call(ptr, empty_tuple, NULL); + Py_DECREF(empty_tuple); + Py_DECREF(ptr); + if (!PyLong_Check(ret)) {{ + PyErr_SetString(PyExc_TypeError, "data_ptr method of Pointer object must return 64-bit int"); + ptr_info.valid = false; + return ptr_info; + }} + ptr_info.dev_ptr = PyLong_AsUnsignedLongLong(ret); + if(!ptr_info.dev_ptr) + return ptr_info; + uint64_t dev_ptr; + int status = cuPointerGetAttribute(&dev_ptr, CU_POINTER_ATTRIBUTE_DEVICE_POINTER, ptr_info.dev_ptr); + if (status == CUDA_ERROR_INVALID_VALUE) {{ + PyErr_Format(PyExc_ValueError, + "Pointer argument (at %d) cannot be accessed from Triton (cpu tensor?)", idx); + ptr_info.valid = false; + }} else if (status != CUDA_SUCCESS) {{ + CUDA_CHECK(status); // Catch any other cuda API errors + ptr_info.valid = false; + }} + ptr_info.dev_ptr = dev_ptr; + Py_DECREF(ret); // Thanks ChatGPT! + return ptr_info; + }} + PyErr_SetString(PyExc_TypeError, "Pointer argument must be either uint64 or have data_ptr method"); + ptr_info.valid = false; + return ptr_info; +}} + +static inline CUtensorMap* getTmaDesc(PyObject *obj) {{ + if (sizeof(CUtensorMap*) != 8) {{ + PyErr_SetString(PyExc_SystemError, "getTmaDesc() requires 64-bit compilation"); + return NULL; + }} + + PyObject *method_handle = PyObject_GetAttrString(obj, "tma_desc_cpu_ptr"); + if (!method_handle) {{ + PyErr_SetString(PyExc_TypeError, "tma_desc_cpu_ptr() method does not exist"); + return NULL; + }} + + PyObject *empty_tuple = PyTuple_New(0); + if (!empty_tuple) {{ + Py_DECREF(method_handle); + PyErr_SetString(PyExc_SystemError, "Internal Python error!"); + return NULL; + }} + PyObject *method_ret = PyObject_Call(method_handle, empty_tuple, NULL); + Py_DECREF(empty_tuple); + Py_DECREF(method_handle); + if (!method_ret) {{ + PyErr_SetString(PyExc_SystemError, "Internal Python error!"); + return NULL; + }} + + if (!PyLong_Check(method_ret)) {{ + PyErr_SetString(PyExc_TypeError, "tma_desc_cpu_ptr() must return 64-bit int"); + Py_DECREF(method_ret); + return NULL; + }} + + uint64_t ptr_as_uint = PyLong_AsUnsignedLongLong(method_ret); + Py_DECREF(method_ret); + if (!ptr_as_uint) {{ + PyErr_SetString(PyExc_ValueError, "received NULL ptr from tma_desc_cpu_ptr()"); + return NULL; + }} + if (ptr_as_uint % 64 != 0) {{ + PyErr_SetString(PyExc_ValueError, "tma_desc_cpu_ptr() must be 64-byte aligned"); + return NULL; + }} + + return (CUtensorMap*)(ptr_as_uint); +}} + +static void ensureCudaContext() {{ + CUcontext pctx; + CUDA_CHECK(cuCtxGetCurrent(&pctx)); + if (!pctx) {{ + // Ensure device context. + CUdevice device; + CUDA_CHECK(cuDeviceGet(&device, 0)); + CUDA_CHECK(cuDevicePrimaryCtxRetain(&pctx, device)); + CUDA_CHECK(cuCtxSetCurrent(pctx)); + }} +}} + +static PyObject* launch(PyObject* self, PyObject* args) {{ + // ensure cuda context is valid before calling any CUDA APIs, e.g. before getPointer calls cuPointerGetAttributes + ensureCudaContext(); + + int gridX, gridY, gridZ; + uint64_t _stream; + uint64_t _function; + PyObject *launch_enter_hook = NULL; + PyObject *launch_exit_hook = NULL; + PyObject *kernel_metadata = NULL; + PyObject *launch_metadata = NULL; + {' '.join([f"{_extracted_type(ty)} _arg{i}; " for i, ty in signature.items()])} + if(!PyArg_ParseTuple(args, \"{format}\", &gridX, &gridY, &gridZ, &_stream, &_function, + &kernel_metadata, &launch_metadata, + &launch_enter_hook, &launch_exit_hook {args_list})) {{ + return NULL; + }} + + int num_warps, num_ctas, shared_memory, clusterDimX, clusterDimY, clusterDimZ; + if (!PyArg_ParseTuple(kernel_metadata, \"iiiiii\", &num_warps, &num_ctas, &shared_memory, &clusterDimX, &clusterDimY, &clusterDimZ)) {{ + PyErr_SetString(PyExc_TypeError, "kernel_metadata must be a tuple"); + return NULL; + }} + + // extract launch metadata + if (launch_enter_hook != Py_None){{ + PyObject* args = Py_BuildValue("(O)", launch_metadata); + PyObject* ret = PyObject_CallObject(launch_enter_hook, args); + Py_DECREF(args); + if (!ret) + return NULL; + }} + + // raise exception asap + {"".join([f"DevicePtrInfo ptr_info{i} = getPointer(_arg{i}, {i}); if (!ptr_info{i}.valid) return NULL;" if ty[0] == "*" else "" for i, ty in signature.items()])}; + {"".join([f"CUtensorMap* tma_ptr{i} = getTmaDesc(_arg{i}); if (!tma_ptr{i}) return NULL;" if ty == "nvTmaDesc" else "" for i, ty in signature.items()])}; + Py_BEGIN_ALLOW_THREADS; + _launch(gridX, gridY, gridZ, num_warps, num_ctas, clusterDimX, clusterDimY, clusterDimZ, shared_memory, (CUstream)_stream, (CUfunction)_function{', ' + ', '.join(internal_args_list) if len(internal_args_list) > 0 else ''}); + Py_END_ALLOW_THREADS; + if (PyErr_Occurred()) {{ + return NULL; + }} + + if(launch_exit_hook != Py_None){{ + PyObject* args = Py_BuildValue("(O)", launch_metadata); + PyObject* ret = PyObject_CallObject(launch_exit_hook, args); + Py_DECREF(args); + if (!ret) + return NULL; + + }} + + // return None + Py_INCREF(Py_None); + return Py_None; +}} + +static PyMethodDef ModuleMethods[] = {{ + {{"launch", launch, METH_VARARGS, "Entry point for all kernels with this signature"}}, + {{NULL, NULL, 0, NULL}} // sentinel +}}; + +static struct PyModuleDef ModuleDef = {{ + PyModuleDef_HEAD_INIT, + \"__triton_launcher\", + NULL, //documentation + -1, //size + ModuleMethods +}}; + +PyMODINIT_FUNC PyInit___triton_launcher(void) {{ + PyObject *m = PyModule_Create(&ModuleDef); + if(m == NULL) {{ + return NULL; + }} + PyModule_AddFunctions(m, ModuleMethods); + return m; +}} +""" + return src + + +class CudaLauncher(object): + + def __init__(self, src, metadata): + ids = {"ids_of_const_exprs": src.fn.constexprs if hasattr(src, "fn") else tuple()} + constants = src.constants if hasattr(src, "constants") else dict() + cst_key = lambda i: src.fn.arg_names.index(i) if isinstance(i, str) else i + constants = {cst_key(key): value for key, value in constants.items()} + signature = {cst_key(key): value for key, value in src.signature.items()} + src = make_launcher(constants, signature, ids) + mod = compile_module_from_src(src, "__triton_launcher") + self.launch = mod.launch + + def __call__(self, *args, **kwargs): + self.launch(*args, **kwargs) + + +class CudaDriver(GPUDriver): + + def __init__(self): + self.utils = CudaUtils() # TODO: make static + self.launcher_cls = CudaLauncher + super().__init__() + + def get_current_target(self): + device = self.get_current_device() + capability = self.get_device_capability(device) + capability = capability[0] * 10 + capability[1] + warp_size = 32 + return GPUTarget("cuda", capability, warp_size) + + def get_device_interface(self): + import torch + return torch.cuda + + @staticmethod + def is_active(): + import torch + return torch.cuda.is_available() and (torch.version.hip is None) + + def get_benchmarker(self): + from triton.testing import do_bench + return do_bench + + def get_empty_cache_for_benchmark(self): + import torch + + # We maintain a buffer of 256 MB that we clear + # before each kernel call to make sure that the L2 cache + # doesn't contain any input data before the run + cache_size = 256 * 1024 * 1024 + return torch.empty(int(cache_size // 4), dtype=torch.int, device='cuda') diff --git a/lib/python3.12/site-packages/triton/backends/nvidia/include/cupti_pcsampling_util.h b/lib/python3.12/site-packages/triton/backends/nvidia/include/cupti_pcsampling_util.h new file mode 100644 index 0000000000000000000000000000000000000000..595d6028fbf2ff9a3bbffaafe90ec80f7d512533 --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/nvidia/include/cupti_pcsampling_util.h @@ -0,0 +1,402 @@ +#if !defined(_CUPTI_PCSAMPLING_UTIL_H_) +#define _CUPTI_PCSAMPLING_UTIL_H_ + +#include +#include + +#include + +#ifndef CUPTI_UTIL_STRUCT_SIZE +#define CUPTI_UTIL_STRUCT_SIZE(type_, lastfield_) (offsetof(type_, lastfield_) + sizeof(((type_*)0)->lastfield_)) +#endif + +#ifndef CHECK_PC_SAMPLING_STRUCT_FIELD_EXISTS +#define CHECK_PC_SAMPLING_STRUCT_FIELD_EXISTS(type, member, structSize) \ + (offsetof(type, member) < structSize) +#endif + +#if defined(__cplusplus) +extern "C" { +#endif + +#if defined(__GNUC__) + #pragma GCC visibility push(default) +#endif + +namespace CUPTI { namespace PcSamplingUtil { + +/** + * \defgroup CUPTI_PCSAMPLING_UTILITY CUPTI PC Sampling Utility API + * Functions, types, and enums that implement the CUPTI PC Sampling Utility API. + * @{ + */ + +/** + * \brief Header info will be stored in file. + */ +typedef struct PACKED_ALIGNMENT { + /** + * Version of file format. + */ + uint32_t version; + /** + * Total number of buffers present in the file. + */ + uint32_t totalBuffers; +} Header; + +/** + * \brief BufferInfo will be stored in the file for every buffer + * i.e for every call of UtilDumpPcSamplingBufferInFile() API. + */ +typedef struct PACKED_ALIGNMENT { + /** + * Total number of PC records. + */ + uint64_t recordCount; + /** + * Count of all stall reasons supported on the GPU + */ + size_t numStallReasons; + /** + * Total number of stall reasons in single record. + */ + uint64_t numSelectedStallReasons; + /** + * Buffer size in Bytes. + */ + uint64_t bufferByteSize; +} BufferInfo; + +/** + * \brief All available stall reasons name and respective indexes + * will be stored in it. + */ +typedef struct PACKED_ALIGNMENT { + /** + * Number of all available stall reasons + */ + size_t numStallReasons; + /** + * Stall reasons names of all available stall reasons + */ + char **stallReasons; + /** + * Stall reason index of all available stall reasons + */ + uint32_t *stallReasonIndex; +} PcSamplingStallReasons; + +/** + * \brief CUPTI PC sampling buffer types. + * + */ +typedef enum { + /** + * Invalid buffer type. + */ + PC_SAMPLING_BUFFER_INVALID = 0, + /** + * Refers to CUpti_PCSamplingData buffer. + */ + PC_SAMPLING_BUFFER_PC_TO_COUNTER_DATA = 1 +} PcSamplingBufferType; + +/** + * \brief CUPTI PC sampling utility API result codes. + * + * Error and result codes returned by CUPTI PC sampling utility API. + */ +typedef enum { + /** + * No error + */ + CUPTI_UTIL_SUCCESS = 0, + /** + * One or more of the parameters are invalid. + */ + CUPTI_UTIL_ERROR_INVALID_PARAMETER = 1, + /** + * Unable to create a new file + */ + CUPTI_UTIL_ERROR_UNABLE_TO_CREATE_FILE = 2, + /** + * Unable to open a file + */ + CUPTI_UTIL_ERROR_UNABLE_TO_OPEN_FILE = 3, + /** + * Read or write operation failed + */ + CUPTI_UTIL_ERROR_READ_WRITE_OPERATION_FAILED = 4, + /** + * Provided file handle is corrupted. + */ + CUPTI_UTIL_ERROR_FILE_HANDLE_CORRUPTED = 5, + /** + * seek operation failed. + */ + CUPTI_UTIL_ERROR_SEEK_OPERATION_FAILED = 6, + /** + * Unable to allocate enough memory to perform the requested + * operation. + */ + CUPTI_UTIL_ERROR_OUT_OF_MEMORY = 7, + /** + * An unknown internal error has occurred. + */ + CUPTI_UTIL_ERROR_UNKNOWN = 999, + CUPTI_UTIL_ERROR_FORCE_INT = 0x7fffffff +} CUptiUtilResult; + +/** + * \brief Params for \ref CuptiUtilPutPcSampData + */ +typedef struct { + /** + * Size of the data structure i.e. CUpti_PCSamplingDisableParamsSize + * CUPTI client should set the size of the structure. It will be used in CUPTI to check what fields are + * available in the structure. Used to preserve backward compatibility. + */ + size_t size; + /** + * Type of buffer to store in file + */ + PcSamplingBufferType bufferType; + /** + * PC sampling buffer. + */ + void *pSamplingData; + /** + * Number of configured attributes + */ + size_t numAttributes; + /** + * Refer \ref CUpti_PCSamplingConfigurationInfo + * It is expected to provide configuration details of at least + * CUPTI_PC_SAMPLING_CONFIGURATION_ATTR_TYPE_STALL_REASON attribute. + */ + CUpti_PCSamplingConfigurationInfo *pPCSamplingConfigurationInfo; + /** + * Refer \ref PcSamplingStallReasons. + */ + PcSamplingStallReasons *pPcSamplingStallReasons; + /** + * File name to store buffer into it. + */ + const char* fileName; +} CUptiUtil_PutPcSampDataParams; +#define CUptiUtil_PutPcSampDataParamsSize CUPTI_UTIL_STRUCT_SIZE(CUptiUtil_PutPcSampDataParams, fileName) + +/** + * \brief Dump PC sampling data into the file. + * + * This API can be called multiple times. + * It will append buffer in the file. + * For every buffer it will store BufferInfo + * so that before retrieving data it will help to allocate buffer + * to store retrieved data. + * This API creates file if file does not present. + * If stallReasonIndex or stallReasons pointer of \ref CUptiUtil_PutPcSampDataParams is NULL + * then stall reasons data will not be stored in file. + * It is expected to store all available stall reason data at least once to refer it during + * offline correlation. + * + * \retval CUPTI_UTIL_SUCCESS + * \retval CUPTI_UTIL_ERROR_INVALID_PARAMETER error out if buffer type is invalid + * or if either of pSamplingData, pParams pointer is NULL or stall reason configuration details not provided + * or filename is empty. + * \retval CUPTI_UTIL_ERROR_UNABLE_TO_CREATE_FILE + * \retval CUPTI_UTIL_ERROR_UNABLE_TO_OPEN_FILE + * \retval CUPTI_UTIL_ERROR_READ_WRITE_OPERATION_FAILED + */ +CUptiUtilResult CUPTIUTILAPI CuptiUtilPutPcSampData(CUptiUtil_PutPcSampDataParams *pParams); + +/** + * \brief Params for \ref CuptiUtilGetHeaderData + */ +typedef struct { + /** + * Size of the data structure i.e. CUpti_PCSamplingDisableParamsSize + * CUPTI client should set the size of the structure. It will be used in CUPTI to check what fields are + * available in the structure. Used to preserve backward compatibility. + */ + size_t size; + /** + * File handle. + */ + std::ifstream *fileHandler; + /** + * Header Info. + */ + Header headerInfo; + +} CUptiUtil_GetHeaderDataParams; +#define CUptiUtil_GetHeaderDataParamsSize CUPTI_UTIL_STRUCT_SIZE(CUptiUtil_GetHeaderDataParams, headerInfo) + +/** + * \brief Get header data of file. + * + * This API must be called once initially while retrieving data from file. + * \ref Header structure, it gives info about total number + * of buffers present in the file. + * + * \retval CUPTI_UTIL_SUCCESS + * \retval CUPTI_UTIL_ERROR_INVALID_PARAMETER error out if either of pParam or fileHandle is NULL or param struct size is incorrect. + * \retval CUPTI_UTIL_ERROR_FILE_HANDLE_CORRUPTED file handle is not in good state to read data from file + * \retval CUPTI_UTIL_ERROR_READ_WRITE_OPERATION_FAILED failed to read data from file. + */ +CUptiUtilResult CUPTIUTILAPI CuptiUtilGetHeaderData(CUptiUtil_GetHeaderDataParams *pParams); + +/** + * \brief Params for \ref CuptiUtilGetBufferInfo + */ +typedef struct { + /** + * Size of the data structure i.e. CUpti_PCSamplingDisableParamsSize + * CUPTI client should set the size of the structure. It will be used in CUPTI to check what fields are + * available in the structure. Used to preserve backward compatibility. + */ + size_t size; + /** + * File handle. + */ + std::ifstream *fileHandler; + /** + * Buffer Info. + */ + BufferInfo bufferInfoData; +} CUptiUtil_GetBufferInfoParams; +#define CUptiUtil_GetBufferInfoParamsSize CUPTI_UTIL_STRUCT_SIZE(CUptiUtil_GetBufferInfoParams, bufferInfoData) + +/** + * \brief Get buffer info data of file. + * + * This API must be called every time before calling CuptiUtilGetPcSampData API. + * \ref BufferInfo structure, it gives info about recordCount and stallReasonCount + * of every record in the buffer. This will help to allocate exact buffer to retrieve data into it. + * + * \retval CUPTI_UTIL_SUCCESS + * \retval CUPTI_UTIL_ERROR_INVALID_PARAMETER error out if either of pParam or fileHandle is NULL or param struct size is incorrect. + * \retval CUPTI_UTIL_ERROR_FILE_HANDLE_CORRUPTED file handle is not in good state to read data from file. + * \retval CUPTI_UTIL_ERROR_READ_WRITE_OPERATION_FAILED failed to read data from file. + */ +CUptiUtilResult CUPTIUTILAPI CuptiUtilGetBufferInfo(CUptiUtil_GetBufferInfoParams *pParams); + +/** + * \brief Params for \ref CuptiUtilGetPcSampData + */ +typedef struct { + /** + * Size of the data structure i.e. CUpti_PCSamplingDisableParamsSize + * CUPTI client should set the size of the structure. It will be used in CUPTI to check what fields are + * available in the structure. Used to preserve backward compatibility. + */ + size_t size; + /** + * File handle. + */ + std::ifstream *fileHandler; + /** + * Type of buffer to store in file + */ + PcSamplingBufferType bufferType; + /** + * Pointer to collected buffer info using \ref CuptiUtilGetBufferInfo + */ + BufferInfo *pBufferInfoData; + /** + * Pointer to allocated memory to store retrieved data from file. + */ + void *pSamplingData; + /** + * Number of configuration attributes + */ + size_t numAttributes; + /** + * Refer \ref CUpti_PCSamplingConfigurationInfo + */ + CUpti_PCSamplingConfigurationInfo *pPCSamplingConfigurationInfo; + /** + * Refer \ref PcSamplingStallReasons. + * For stallReasons field of \ref PcSamplingStallReasons it is expected to + * allocate memory for each string element of array. + */ + PcSamplingStallReasons *pPcSamplingStallReasons; +} CUptiUtil_GetPcSampDataParams; +#define CUptiUtil_GetPcSampDataParamsSize CUPTI_UTIL_STRUCT_SIZE(CUptiUtil_GetPcSampDataParams, pPcSamplingStallReasons) + +/** + * \brief Retrieve PC sampling data from file into allocated buffer. + * + * This API must be called after CuptiUtilGetBufferInfo API. + * It will retrieve data from file into allocated buffer. + * + * \retval CUPTI_UTIL_SUCCESS + * \retval CUPTI_UTIL_ERROR_INVALID_PARAMETER error out if buffer type is invalid + * or if either of pSampData, pParams is NULL. If pPcSamplingStallReasons is not NULL then + * error out if either of stallReasonIndex, stallReasons or stallReasons array element pointer is NULL. + * or filename is empty. + * \retval CUPTI_UTIL_ERROR_READ_WRITE_OPERATION_FAILED + * \retval CUPTI_UTIL_ERROR_FILE_HANDLE_CORRUPTED file handle is not in good state to read data from file. + */ +CUptiUtilResult CUPTIUTILAPI CuptiUtilGetPcSampData(CUptiUtil_GetPcSampDataParams *pParams); + +/** + * \brief Params for \ref CuptiUtilMergePcSampData + */ +typedef struct +{ + /** + * Size of the data structure i.e. CUpti_PCSamplingDisableParamsSize + * CUPTI client should set the size of the structure. It will be used in CUPTI to check what fields are + * available in the structure. Used to preserve backward compatibility. + */ + size_t size; + /** + * Number of buffers to merge. + */ + size_t numberOfBuffers; + /** + * Pointer to array of buffers to merge + */ + CUpti_PCSamplingData *PcSampDataBuffer; + /** + * Pointer to array of merged buffers as per the range id. + */ + CUpti_PCSamplingData **MergedPcSampDataBuffers; + /** + * Number of merged buffers. + */ + size_t *numMergedBuffer; +} CUptiUtil_MergePcSampDataParams; +#define CUptiUtil_MergePcSampDataParamsSize CUPTI_UTIL_STRUCT_SIZE(CUptiUtil_MergePcSampDataParams, numMergedBuffer) + +/** + * \brief Merge PC sampling data range id wise. + * + * This API merge PC sampling data range id wise. + * It allocates memory for merged data and fill data in it + * and provide buffer pointer in MergedPcSampDataBuffers field. + * It is expected from user to free merge data buffers after use. + * + * \retval CUPTI_UTIL_SUCCESS + * \retval CUPTI_UTIL_ERROR_INVALID_PARAMETER error out if param struct size is invalid + * or count of buffers to merge is invalid i.e less than 1 + * or either of PcSampDataBuffer, MergedPcSampDataBuffers, numMergedBuffer is NULL + * \retval CUPTI_UTIL_ERROR_OUT_OF_MEMORY Unable to allocate memory for merged buffer. + */ +CUptiUtilResult CUPTIUTILAPI CuptiUtilMergePcSampData(CUptiUtil_MergePcSampDataParams *pParams); + +/** @} */ /* END CUPTI_PCSAMPLING_UTILITY */ + +} } + +#if defined(__GNUC__) + #pragma GCC visibility pop +#endif + +#if defined(__cplusplus) +} +#endif + +#endif diff --git a/lib/python3.12/site-packages/triton/backends/nvidia/include/driver_types.h b/lib/python3.12/site-packages/triton/backends/nvidia/include/driver_types.h new file mode 100644 index 0000000000000000000000000000000000000000..b5c515fd3584ae6546638dc52c11e3ce6a383049 --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/nvidia/include/driver_types.h @@ -0,0 +1,3642 @@ +/* + * Copyright 1993-2023 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO LICENSEE: + * + * This source code and/or documentation ("Licensed Deliverables") are + * subject to NVIDIA intellectual property rights under U.S. and + * international Copyright laws. + * + * These Licensed Deliverables contained herein is PROPRIETARY and + * CONFIDENTIAL to NVIDIA and is being provided under the terms and + * conditions of a form of NVIDIA software license agreement by and + * between NVIDIA and Licensee ("License Agreement") or electronically + * accepted by Licensee. Notwithstanding any terms or conditions to + * the contrary in the License Agreement, reproduction or disclosure + * of the Licensed Deliverables to any third party without the express + * written consent of NVIDIA is prohibited. + * + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE + * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS + * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. + * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED + * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, + * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY + * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THESE LICENSED DELIVERABLES. + * + * U.S. Government End Users. These Licensed Deliverables are a + * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT + * 1995), consisting of "commercial computer software" and "commercial + * computer software documentation" as such terms are used in 48 + * C.F.R. 12.212 (SEPT 1995) and is provided to the U.S. Government + * only as a commercial end item. Consistent with 48 C.F.R.12.212 and + * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all + * U.S. Government End Users acquire the Licensed Deliverables with + * only those rights set forth herein. + * + * Any use of the Licensed Deliverables in individual and commercial + * software must include, in the user documentation and internal + * comments to the code, the above Disclaimer and U.S. Government End + * Users Notice. + */ + +#if !defined(__DRIVER_TYPES_H__) +#define __DRIVER_TYPES_H__ + +#if !defined(__CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__) +#define __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__ +#define __UNDEF_CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS_DRIVER_TYPES_H__ +#endif + +#ifndef __DOXYGEN_ONLY__ +#include "crt/host_defines.h" +#endif +#include "vector_types.h" + + + +#ifndef __CUDACC_RTC_MINIMAL__ +/** + * \defgroup CUDART_TYPES Data types used by CUDA Runtime + * \ingroup CUDART + * + * @{ + */ + +/******************************************************************************* +* * +* TYPE DEFINITIONS USED BY RUNTIME API * +* * +*******************************************************************************/ + +#if !defined(__CUDA_INTERNAL_COMPILATION__) + + +#if !defined(__CUDACC_RTC__) +#include +#include +#endif /* !defined(__CUDACC_RTC__) */ + +#define cudaHostAllocDefault 0x00 /**< Default page-locked allocation flag */ +#define cudaHostAllocPortable 0x01 /**< Pinned memory accessible by all CUDA contexts */ +#define cudaHostAllocMapped 0x02 /**< Map allocation into device space */ +#define cudaHostAllocWriteCombined 0x04 /**< Write-combined memory */ + +#define cudaHostRegisterDefault 0x00 /**< Default host memory registration flag */ +#define cudaHostRegisterPortable 0x01 /**< Pinned memory accessible by all CUDA contexts */ +#define cudaHostRegisterMapped 0x02 /**< Map registered memory into device space */ +#define cudaHostRegisterIoMemory 0x04 /**< Memory-mapped I/O space */ +#define cudaHostRegisterReadOnly 0x08 /**< Memory-mapped read-only */ + +#define cudaPeerAccessDefault 0x00 /**< Default peer addressing enable flag */ + +#define cudaStreamDefault 0x00 /**< Default stream flag */ +#define cudaStreamNonBlocking 0x01 /**< Stream does not synchronize with stream 0 (the NULL stream) */ + + /** + * Legacy stream handle + * + * Stream handle that can be passed as a cudaStream_t to use an implicit stream + * with legacy synchronization behavior. + * + * See details of the \link_sync_behavior + */ +#define cudaStreamLegacy ((cudaStream_t)0x1) + +/** + * Per-thread stream handle + * + * Stream handle that can be passed as a cudaStream_t to use an implicit stream + * with per-thread synchronization behavior. + * + * See details of the \link_sync_behavior + */ +#define cudaStreamPerThread ((cudaStream_t)0x2) + +#define cudaEventDefault 0x00 /**< Default event flag */ +#define cudaEventBlockingSync 0x01 /**< Event uses blocking synchronization */ +#define cudaEventDisableTiming 0x02 /**< Event will not record timing data */ +#define cudaEventInterprocess 0x04 /**< Event is suitable for interprocess use. cudaEventDisableTiming must be set */ + +#define cudaEventRecordDefault 0x00 /**< Default event record flag */ +#define cudaEventRecordExternal 0x01 /**< Event is captured in the graph as an external event node when performing stream capture */ + +#define cudaEventWaitDefault 0x00 /**< Default event wait flag */ +#define cudaEventWaitExternal 0x01 /**< Event is captured in the graph as an external event node when performing stream capture */ + +#define cudaDeviceScheduleAuto 0x00 /**< Device flag - Automatic scheduling */ +#define cudaDeviceScheduleSpin 0x01 /**< Device flag - Spin default scheduling */ +#define cudaDeviceScheduleYield 0x02 /**< Device flag - Yield default scheduling */ +#define cudaDeviceScheduleBlockingSync 0x04 /**< Device flag - Use blocking synchronization */ +#define cudaDeviceBlockingSync 0x04 /**< Device flag - Use blocking synchronization + * \deprecated This flag was deprecated as of CUDA 4.0 and + * replaced with ::cudaDeviceScheduleBlockingSync. */ +#define cudaDeviceScheduleMask 0x07 /**< Device schedule flags mask */ +#define cudaDeviceMapHost 0x08 /**< Device flag - Support mapped pinned allocations */ +#define cudaDeviceLmemResizeToMax 0x10 /**< Device flag - Keep local memory allocation after launch */ +#define cudaDeviceSyncMemops 0x80 /**< Device flag - Ensure synchronous memory operations on this context will synchronize */ +#define cudaDeviceMask 0xff /**< Device flags mask */ + +#define cudaArrayDefault 0x00 /**< Default CUDA array allocation flag */ +#define cudaArrayLayered 0x01 /**< Must be set in cudaMalloc3DArray to create a layered CUDA array */ +#define cudaArraySurfaceLoadStore 0x02 /**< Must be set in cudaMallocArray or cudaMalloc3DArray in order to bind surfaces to the CUDA array */ +#define cudaArrayCubemap 0x04 /**< Must be set in cudaMalloc3DArray to create a cubemap CUDA array */ +#define cudaArrayTextureGather 0x08 /**< Must be set in cudaMallocArray or cudaMalloc3DArray in order to perform texture gather operations on the CUDA array */ +#define cudaArrayColorAttachment 0x20 /**< Must be set in cudaExternalMemoryGetMappedMipmappedArray if the mipmapped array is used as a color target in a graphics API */ +#define cudaArraySparse 0x40 /**< Must be set in cudaMallocArray, cudaMalloc3DArray or cudaMallocMipmappedArray in order to create a sparse CUDA array or CUDA mipmapped array */ +#define cudaArrayDeferredMapping 0x80 /**< Must be set in cudaMallocArray, cudaMalloc3DArray or cudaMallocMipmappedArray in order to create a deferred mapping CUDA array or CUDA mipmapped array */ + +#define cudaIpcMemLazyEnablePeerAccess 0x01 /**< Automatically enable peer access between remote devices as needed */ + +#define cudaMemAttachGlobal 0x01 /**< Memory can be accessed by any stream on any device*/ +#define cudaMemAttachHost 0x02 /**< Memory cannot be accessed by any stream on any device */ +#define cudaMemAttachSingle 0x04 /**< Memory can only be accessed by a single stream on the associated device */ + +#define cudaOccupancyDefault 0x00 /**< Default behavior */ +#define cudaOccupancyDisableCachingOverride 0x01 /**< Assume global caching is enabled and cannot be automatically turned off */ + +#define cudaCpuDeviceId ((int)-1) /**< Device id that represents the CPU */ +#define cudaInvalidDeviceId ((int)-2) /**< Device id that represents an invalid device */ +#define cudaInitDeviceFlagsAreValid 0x01 /**< Tell the CUDA runtime that DeviceFlags is being set in cudaInitDevice call */ +/** + * If set, each kernel launched as part of ::cudaLaunchCooperativeKernelMultiDevice only + * waits for prior work in the stream corresponding to that GPU to complete before the + * kernel begins execution. + */ +#define cudaCooperativeLaunchMultiDeviceNoPreSync 0x01 + +/** + * If set, any subsequent work pushed in a stream that participated in a call to + * ::cudaLaunchCooperativeKernelMultiDevice will only wait for the kernel launched on + * the GPU corresponding to that stream to complete before it begins execution. + */ +#define cudaCooperativeLaunchMultiDeviceNoPostSync 0x02 + +#endif /* !__CUDA_INTERNAL_COMPILATION__ */ + +/** \cond impl_private */ +#if defined(__DOXYGEN_ONLY__) || defined(CUDA_ENABLE_DEPRECATED) +#define __CUDA_DEPRECATED +#elif defined(_MSC_VER) +#define __CUDA_DEPRECATED __declspec(deprecated) +#elif defined(__GNUC__) +#define __CUDA_DEPRECATED __attribute__((deprecated)) +#else +#define __CUDA_DEPRECATED +#endif +/** \endcond impl_private */ + +/******************************************************************************* +* * +* * +* * +*******************************************************************************/ + +/** + * CUDA error types + */ +enum __device_builtin__ cudaError +{ + /** + * The API call returned with no errors. In the case of query calls, this + * also means that the operation being queried is complete (see + * ::cudaEventQuery() and ::cudaStreamQuery()). + */ + cudaSuccess = 0, + + /** + * This indicates that one or more of the parameters passed to the API call + * is not within an acceptable range of values. + */ + cudaErrorInvalidValue = 1, + + /** + * The API call failed because it was unable to allocate enough memory or + * other resources to perform the requested operation. + */ + cudaErrorMemoryAllocation = 2, + + /** + * The API call failed because the CUDA driver and runtime could not be + * initialized. + */ + cudaErrorInitializationError = 3, + + /** + * This indicates that a CUDA Runtime API call cannot be executed because + * it is being called during process shut down, at a point in time after + * CUDA driver has been unloaded. + */ + cudaErrorCudartUnloading = 4, + + /** + * This indicates profiler is not initialized for this run. This can + * happen when the application is running with external profiling tools + * like visual profiler. + */ + cudaErrorProfilerDisabled = 5, + + /** + * \deprecated + * This error return is deprecated as of CUDA 5.0. It is no longer an error + * to attempt to enable/disable the profiling via ::cudaProfilerStart or + * ::cudaProfilerStop without initialization. + */ + cudaErrorProfilerNotInitialized = 6, + + /** + * \deprecated + * This error return is deprecated as of CUDA 5.0. It is no longer an error + * to call cudaProfilerStart() when profiling is already enabled. + */ + cudaErrorProfilerAlreadyStarted = 7, + + /** + * \deprecated + * This error return is deprecated as of CUDA 5.0. It is no longer an error + * to call cudaProfilerStop() when profiling is already disabled. + */ + cudaErrorProfilerAlreadyStopped = 8, + /** + * This indicates that a kernel launch is requesting resources that can + * never be satisfied by the current device. Requesting more shared memory + * per block than the device supports will trigger this error, as will + * requesting too many threads or blocks. See ::cudaDeviceProp for more + * device limitations. + */ + cudaErrorInvalidConfiguration = 9, + + /** + * This indicates that one or more of the pitch-related parameters passed + * to the API call is not within the acceptable range for pitch. + */ + cudaErrorInvalidPitchValue = 12, + + /** + * This indicates that the symbol name/identifier passed to the API call + * is not a valid name or identifier. + */ + cudaErrorInvalidSymbol = 13, + + /** + * This indicates that at least one host pointer passed to the API call is + * not a valid host pointer. + * \deprecated + * This error return is deprecated as of CUDA 10.1. + */ + cudaErrorInvalidHostPointer = 16, + + /** + * This indicates that at least one device pointer passed to the API call is + * not a valid device pointer. + * \deprecated + * This error return is deprecated as of CUDA 10.1. + */ + cudaErrorInvalidDevicePointer = 17, + /** + * This indicates that the texture passed to the API call is not a valid + * texture. + */ + cudaErrorInvalidTexture = 18, + + /** + * This indicates that the texture binding is not valid. This occurs if you + * call ::cudaGetTextureAlignmentOffset() with an unbound texture. + */ + cudaErrorInvalidTextureBinding = 19, + + /** + * This indicates that the channel descriptor passed to the API call is not + * valid. This occurs if the format is not one of the formats specified by + * ::cudaChannelFormatKind, or if one of the dimensions is invalid. + */ + cudaErrorInvalidChannelDescriptor = 20, + + /** + * This indicates that the direction of the memcpy passed to the API call is + * not one of the types specified by ::cudaMemcpyKind. + */ + cudaErrorInvalidMemcpyDirection = 21, + + /** + * This indicated that the user has taken the address of a constant variable, + * which was forbidden up until the CUDA 3.1 release. + * \deprecated + * This error return is deprecated as of CUDA 3.1. Variables in constant + * memory may now have their address taken by the runtime via + * ::cudaGetSymbolAddress(). + */ + cudaErrorAddressOfConstant = 22, + + /** + * This indicated that a texture fetch was not able to be performed. + * This was previously used for device emulation of texture operations. + * \deprecated + * This error return is deprecated as of CUDA 3.1. Device emulation mode was + * removed with the CUDA 3.1 release. + */ + cudaErrorTextureFetchFailed = 23, + + /** + * This indicated that a texture was not bound for access. + * This was previously used for device emulation of texture operations. + * \deprecated + * This error return is deprecated as of CUDA 3.1. Device emulation mode was + * removed with the CUDA 3.1 release. + */ + cudaErrorTextureNotBound = 24, + + /** + * This indicated that a synchronization operation had failed. + * This was previously used for some device emulation functions. + * \deprecated + * This error return is deprecated as of CUDA 3.1. Device emulation mode was + * removed with the CUDA 3.1 release. + */ + cudaErrorSynchronizationError = 25, + /** + * This indicates that a non-float texture was being accessed with linear + * filtering. This is not supported by CUDA. + */ + cudaErrorInvalidFilterSetting = 26, + + /** + * This indicates that an attempt was made to read a non-float texture as a + * normalized float. This is not supported by CUDA. + */ + cudaErrorInvalidNormSetting = 27, + + /** + * Mixing of device and device emulation code was not allowed. + * \deprecated + * This error return is deprecated as of CUDA 3.1. Device emulation mode was + * removed with the CUDA 3.1 release. + */ + cudaErrorMixedDeviceExecution = 28, + + /** + * This indicates that the API call is not yet implemented. Production + * releases of CUDA will never return this error. + * \deprecated + * This error return is deprecated as of CUDA 4.1. + */ + cudaErrorNotYetImplemented = 31, + + /** + * This indicated that an emulated device pointer exceeded the 32-bit address + * range. + * \deprecated + * This error return is deprecated as of CUDA 3.1. Device emulation mode was + * removed with the CUDA 3.1 release. + */ + cudaErrorMemoryValueTooLarge = 32, + /** + * This indicates that the CUDA driver that the application has loaded is a + * stub library. Applications that run with the stub rather than a real + * driver loaded will result in CUDA API returning this error. + */ + cudaErrorStubLibrary = 34, + + /** + * This indicates that the installed NVIDIA CUDA driver is older than the + * CUDA runtime library. This is not a supported configuration. Users should + * install an updated NVIDIA display driver to allow the application to run. + */ + cudaErrorInsufficientDriver = 35, + + /** + * This indicates that the API call requires a newer CUDA driver than the one + * currently installed. Users should install an updated NVIDIA CUDA driver + * to allow the API call to succeed. + */ + cudaErrorCallRequiresNewerDriver = 36, + + /** + * This indicates that the surface passed to the API call is not a valid + * surface. + */ + cudaErrorInvalidSurface = 37, + + /** + * This indicates that multiple global or constant variables (across separate + * CUDA source files in the application) share the same string name. + */ + cudaErrorDuplicateVariableName = 43, + + /** + * This indicates that multiple textures (across separate CUDA source + * files in the application) share the same string name. + */ + cudaErrorDuplicateTextureName = 44, + + /** + * This indicates that multiple surfaces (across separate CUDA source + * files in the application) share the same string name. + */ + cudaErrorDuplicateSurfaceName = 45, + + /** + * This indicates that all CUDA devices are busy or unavailable at the current + * time. Devices are often busy/unavailable due to use of + * ::cudaComputeModeProhibited, ::cudaComputeModeExclusiveProcess, or when long + * running CUDA kernels have filled up the GPU and are blocking new work + * from starting. They can also be unavailable due to memory constraints + * on a device that already has active CUDA work being performed. + */ + cudaErrorDevicesUnavailable = 46, + + /** + * This indicates that the current context is not compatible with this + * the CUDA Runtime. This can only occur if you are using CUDA + * Runtime/Driver interoperability and have created an existing Driver + * context using the driver API. The Driver context may be incompatible + * either because the Driver context was created using an older version + * of the API, because the Runtime API call expects a primary driver + * context and the Driver context is not primary, or because the Driver + * context has been destroyed. Please see \ref CUDART_DRIVER "Interactions + * with the CUDA Driver API" for more information. + */ + cudaErrorIncompatibleDriverContext = 49, + + /** + * The device function being invoked (usually via ::cudaLaunchKernel()) was not + * previously configured via the ::cudaConfigureCall() function. + */ + cudaErrorMissingConfiguration = 52, + + /** + * This indicated that a previous kernel launch failed. This was previously + * used for device emulation of kernel launches. + * \deprecated + * This error return is deprecated as of CUDA 3.1. Device emulation mode was + * removed with the CUDA 3.1 release. + */ + cudaErrorPriorLaunchFailure = 53, + /** + * This error indicates that a device runtime grid launch did not occur + * because the depth of the child grid would exceed the maximum supported + * number of nested grid launches. + */ + cudaErrorLaunchMaxDepthExceeded = 65, + + /** + * This error indicates that a grid launch did not occur because the kernel + * uses file-scoped textures which are unsupported by the device runtime. + * Kernels launched via the device runtime only support textures created with + * the Texture Object API's. + */ + cudaErrorLaunchFileScopedTex = 66, + + /** + * This error indicates that a grid launch did not occur because the kernel + * uses file-scoped surfaces which are unsupported by the device runtime. + * Kernels launched via the device runtime only support surfaces created with + * the Surface Object API's. + */ + cudaErrorLaunchFileScopedSurf = 67, + + /** + * This error indicates that a call to ::cudaDeviceSynchronize made from + * the device runtime failed because the call was made at grid depth greater + * than than either the default (2 levels of grids) or user specified device + * limit ::cudaLimitDevRuntimeSyncDepth. To be able to synchronize on + * launched grids at a greater depth successfully, the maximum nested + * depth at which ::cudaDeviceSynchronize will be called must be specified + * with the ::cudaLimitDevRuntimeSyncDepth limit to the ::cudaDeviceSetLimit + * api before the host-side launch of a kernel using the device runtime. + * Keep in mind that additional levels of sync depth require the runtime + * to reserve large amounts of device memory that cannot be used for + * user allocations. Note that ::cudaDeviceSynchronize made from device + * runtime is only supported on devices of compute capability < 9.0. + */ + cudaErrorSyncDepthExceeded = 68, + + /** + * This error indicates that a device runtime grid launch failed because + * the launch would exceed the limit ::cudaLimitDevRuntimePendingLaunchCount. + * For this launch to proceed successfully, ::cudaDeviceSetLimit must be + * called to set the ::cudaLimitDevRuntimePendingLaunchCount to be higher + * than the upper bound of outstanding launches that can be issued to the + * device runtime. Keep in mind that raising the limit of pending device + * runtime launches will require the runtime to reserve device memory that + * cannot be used for user allocations. + */ + cudaErrorLaunchPendingCountExceeded = 69, + + /** + * The requested device function does not exist or is not compiled for the + * proper device architecture. + */ + cudaErrorInvalidDeviceFunction = 98, + + /** + * This indicates that no CUDA-capable devices were detected by the installed + * CUDA driver. + */ + cudaErrorNoDevice = 100, + + /** + * This indicates that the device ordinal supplied by the user does not + * correspond to a valid CUDA device or that the action requested is + * invalid for the specified device. + */ + cudaErrorInvalidDevice = 101, + + /** + * This indicates that the device doesn't have a valid Grid License. + */ + cudaErrorDeviceNotLicensed = 102, + + /** + * By default, the CUDA runtime may perform a minimal set of self-tests, + * as well as CUDA driver tests, to establish the validity of both. + * Introduced in CUDA 11.2, this error return indicates that at least one + * of these tests has failed and the validity of either the runtime + * or the driver could not be established. + */ + cudaErrorSoftwareValidityNotEstablished = 103, + + /** + * This indicates an internal startup failure in the CUDA runtime. + */ + cudaErrorStartupFailure = 127, + + /** + * This indicates that the device kernel image is invalid. + */ + cudaErrorInvalidKernelImage = 200, + + /** + * This most frequently indicates that there is no context bound to the + * current thread. This can also be returned if the context passed to an + * API call is not a valid handle (such as a context that has had + * ::cuCtxDestroy() invoked on it). This can also be returned if a user + * mixes different API versions (i.e. 3010 context with 3020 API calls). + * See ::cuCtxGetApiVersion() for more details. + */ + cudaErrorDeviceUninitialized = 201, + + /** + * This indicates that the buffer object could not be mapped. + */ + cudaErrorMapBufferObjectFailed = 205, + + /** + * This indicates that the buffer object could not be unmapped. + */ + cudaErrorUnmapBufferObjectFailed = 206, + + /** + * This indicates that the specified array is currently mapped and thus + * cannot be destroyed. + */ + cudaErrorArrayIsMapped = 207, + + /** + * This indicates that the resource is already mapped. + */ + cudaErrorAlreadyMapped = 208, + + /** + * This indicates that there is no kernel image available that is suitable + * for the device. This can occur when a user specifies code generation + * options for a particular CUDA source file that do not include the + * corresponding device configuration. + */ + cudaErrorNoKernelImageForDevice = 209, + + /** + * This indicates that a resource has already been acquired. + */ + cudaErrorAlreadyAcquired = 210, + + /** + * This indicates that a resource is not mapped. + */ + cudaErrorNotMapped = 211, + + /** + * This indicates that a mapped resource is not available for access as an + * array. + */ + cudaErrorNotMappedAsArray = 212, + + /** + * This indicates that a mapped resource is not available for access as a + * pointer. + */ + cudaErrorNotMappedAsPointer = 213, + + /** + * This indicates that an uncorrectable ECC error was detected during + * execution. + */ + cudaErrorECCUncorrectable = 214, + + /** + * This indicates that the ::cudaLimit passed to the API call is not + * supported by the active device. + */ + cudaErrorUnsupportedLimit = 215, + + /** + * This indicates that a call tried to access an exclusive-thread device that + * is already in use by a different thread. + */ + cudaErrorDeviceAlreadyInUse = 216, + + /** + * This error indicates that P2P access is not supported across the given + * devices. + */ + cudaErrorPeerAccessUnsupported = 217, + + /** + * A PTX compilation failed. The runtime may fall back to compiling PTX if + * an application does not contain a suitable binary for the current device. + */ + cudaErrorInvalidPtx = 218, + + /** + * This indicates an error with the OpenGL or DirectX context. + */ + cudaErrorInvalidGraphicsContext = 219, + + /** + * This indicates that an uncorrectable NVLink error was detected during the + * execution. + */ + cudaErrorNvlinkUncorrectable = 220, + + /** + * This indicates that the PTX JIT compiler library was not found. The JIT Compiler + * library is used for PTX compilation. The runtime may fall back to compiling PTX + * if an application does not contain a suitable binary for the current device. + */ + cudaErrorJitCompilerNotFound = 221, + + /** + * This indicates that the provided PTX was compiled with an unsupported toolchain. + * The most common reason for this, is the PTX was generated by a compiler newer + * than what is supported by the CUDA driver and PTX JIT compiler. + */ + cudaErrorUnsupportedPtxVersion = 222, + + /** + * This indicates that the JIT compilation was disabled. The JIT compilation compiles + * PTX. The runtime may fall back to compiling PTX if an application does not contain + * a suitable binary for the current device. + */ + cudaErrorJitCompilationDisabled = 223, + + /** + * This indicates that the provided execution affinity is not supported by the device. + */ + cudaErrorUnsupportedExecAffinity = 224, + + /** + * This indicates that the code to be compiled by the PTX JIT contains + * unsupported call to cudaDeviceSynchronize. + */ + cudaErrorUnsupportedDevSideSync = 225, + + /** + * This indicates that the device kernel source is invalid. + */ + cudaErrorInvalidSource = 300, + + /** + * This indicates that the file specified was not found. + */ + cudaErrorFileNotFound = 301, + + /** + * This indicates that a link to a shared object failed to resolve. + */ + cudaErrorSharedObjectSymbolNotFound = 302, + + /** + * This indicates that initialization of a shared object failed. + */ + cudaErrorSharedObjectInitFailed = 303, + + /** + * This error indicates that an OS call failed. + */ + cudaErrorOperatingSystem = 304, + + /** + * This indicates that a resource handle passed to the API call was not + * valid. Resource handles are opaque types like ::cudaStream_t and + * ::cudaEvent_t. + */ + cudaErrorInvalidResourceHandle = 400, + + /** + * This indicates that a resource required by the API call is not in a + * valid state to perform the requested operation. + */ + cudaErrorIllegalState = 401, + + /** + * This indicates an attempt was made to introspect an object in a way that + * would discard semantically important information. This is either due to + * the object using funtionality newer than the API version used to + * introspect it or omission of optional return arguments. + */ + cudaErrorLossyQuery = 402, + + /** + * This indicates that a named symbol was not found. Examples of symbols + * are global/constant variable names, driver function names, texture names, + * and surface names. + */ + cudaErrorSymbolNotFound = 500, + + /** + * This indicates that asynchronous operations issued previously have not + * completed yet. This result is not actually an error, but must be indicated + * differently than ::cudaSuccess (which indicates completion). Calls that + * may return this value include ::cudaEventQuery() and ::cudaStreamQuery(). + */ + cudaErrorNotReady = 600, + + /** + * The device encountered a load or store instruction on an invalid memory address. + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorIllegalAddress = 700, + + /** + * This indicates that a launch did not occur because it did not have + * appropriate resources. Although this error is similar to + * ::cudaErrorInvalidConfiguration, this error usually indicates that the + * user has attempted to pass too many arguments to the device kernel, or the + * kernel launch specifies too many threads for the kernel's register count. + */ + cudaErrorLaunchOutOfResources = 701, + + /** + * This indicates that the device kernel took too long to execute. This can + * only occur if timeouts are enabled - see the device property + * \ref ::cudaDeviceProp::kernelExecTimeoutEnabled "kernelExecTimeoutEnabled" + * for more information. + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorLaunchTimeout = 702, + + /** + * This error indicates a kernel launch that uses an incompatible texturing + * mode. + */ + cudaErrorLaunchIncompatibleTexturing = 703, + + /** + * This error indicates that a call to ::cudaDeviceEnablePeerAccess() is + * trying to re-enable peer addressing on from a context which has already + * had peer addressing enabled. + */ + cudaErrorPeerAccessAlreadyEnabled = 704, + + /** + * This error indicates that ::cudaDeviceDisablePeerAccess() is trying to + * disable peer addressing which has not been enabled yet via + * ::cudaDeviceEnablePeerAccess(). + */ + cudaErrorPeerAccessNotEnabled = 705, + + /** + * This indicates that the user has called ::cudaSetValidDevices(), + * ::cudaSetDeviceFlags(), ::cudaD3D9SetDirect3DDevice(), + * ::cudaD3D10SetDirect3DDevice, ::cudaD3D11SetDirect3DDevice(), or + * ::cudaVDPAUSetVDPAUDevice() after initializing the CUDA runtime by + * calling non-device management operations (allocating memory and + * launching kernels are examples of non-device management operations). + * This error can also be returned if using runtime/driver + * interoperability and there is an existing ::CUcontext active on the + * host thread. + */ + cudaErrorSetOnActiveProcess = 708, + + /** + * This error indicates that the context current to the calling thread + * has been destroyed using ::cuCtxDestroy, or is a primary context which + * has not yet been initialized. + */ + cudaErrorContextIsDestroyed = 709, + + /** + * An assert triggered in device code during kernel execution. The device + * cannot be used again. All existing allocations are invalid. To continue + * using CUDA, the process must be terminated and relaunched. + */ + cudaErrorAssert = 710, + + /** + * This error indicates that the hardware resources required to enable + * peer access have been exhausted for one or more of the devices + * passed to ::cudaEnablePeerAccess(). + */ + cudaErrorTooManyPeers = 711, + + /** + * This error indicates that the memory range passed to ::cudaHostRegister() + * has already been registered. + */ + cudaErrorHostMemoryAlreadyRegistered = 712, + + /** + * This error indicates that the pointer passed to ::cudaHostUnregister() + * does not correspond to any currently registered memory region. + */ + cudaErrorHostMemoryNotRegistered = 713, + + /** + * Device encountered an error in the call stack during kernel execution, + * possibly due to stack corruption or exceeding the stack size limit. + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorHardwareStackError = 714, + + /** + * The device encountered an illegal instruction during kernel execution + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorIllegalInstruction = 715, + + /** + * The device encountered a load or store instruction + * on a memory address which is not aligned. + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorMisalignedAddress = 716, + + /** + * While executing a kernel, the device encountered an instruction + * which can only operate on memory locations in certain address spaces + * (global, shared, or local), but was supplied a memory address not + * belonging to an allowed address space. + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorInvalidAddressSpace = 717, + + /** + * The device encountered an invalid program counter. + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorInvalidPc = 718, + + /** + * An exception occurred on the device while executing a kernel. Common + * causes include dereferencing an invalid device pointer and accessing + * out of bounds shared memory. Less common cases can be system specific - more + * information about these cases can be found in the system specific user guide. + * This leaves the process in an inconsistent state and any further CUDA work + * will return the same error. To continue using CUDA, the process must be terminated + * and relaunched. + */ + cudaErrorLaunchFailure = 719, + + /** + * This error indicates that the number of blocks launched per grid for a kernel that was + * launched via either ::cudaLaunchCooperativeKernel or ::cudaLaunchCooperativeKernelMultiDevice + * exceeds the maximum number of blocks as allowed by ::cudaOccupancyMaxActiveBlocksPerMultiprocessor + * or ::cudaOccupancyMaxActiveBlocksPerMultiprocessorWithFlags times the number of multiprocessors + * as specified by the device attribute ::cudaDevAttrMultiProcessorCount. + */ + cudaErrorCooperativeLaunchTooLarge = 720, + + /** + * This error indicates the attempted operation is not permitted. + */ + cudaErrorNotPermitted = 800, + + /** + * This error indicates the attempted operation is not supported + * on the current system or device. + */ + cudaErrorNotSupported = 801, + + /** + * This error indicates that the system is not yet ready to start any CUDA + * work. To continue using CUDA, verify the system configuration is in a + * valid state and all required driver daemons are actively running. + * More information about this error can be found in the system specific + * user guide. + */ + cudaErrorSystemNotReady = 802, + + /** + * This error indicates that there is a mismatch between the versions of + * the display driver and the CUDA driver. Refer to the compatibility documentation + * for supported versions. + */ + cudaErrorSystemDriverMismatch = 803, + + /** + * This error indicates that the system was upgraded to run with forward compatibility + * but the visible hardware detected by CUDA does not support this configuration. + * Refer to the compatibility documentation for the supported hardware matrix or ensure + * that only supported hardware is visible during initialization via the CUDA_VISIBLE_DEVICES + * environment variable. + */ + cudaErrorCompatNotSupportedOnDevice = 804, + + /** + * This error indicates that the MPS client failed to connect to the MPS control daemon or the MPS server. + */ + cudaErrorMpsConnectionFailed = 805, + + /** + * This error indicates that the remote procedural call between the MPS server and the MPS client failed. + */ + cudaErrorMpsRpcFailure = 806, + + /** + * This error indicates that the MPS server is not ready to accept new MPS client requests. + * This error can be returned when the MPS server is in the process of recovering from a fatal failure. + */ + cudaErrorMpsServerNotReady = 807, + + /** + * This error indicates that the hardware resources required to create MPS client have been exhausted. + */ + cudaErrorMpsMaxClientsReached = 808, + + /** + * This error indicates the the hardware resources required to device connections have been exhausted. + */ + cudaErrorMpsMaxConnectionsReached = 809, + + /** + * This error indicates that the MPS client has been terminated by the server. To continue using CUDA, the process must be terminated and relaunched. + */ + cudaErrorMpsClientTerminated = 810, + + /** + * This error indicates, that the program is using CUDA Dynamic Parallelism, but the current configuration, like MPS, does not support it. + */ + cudaErrorCdpNotSupported = 811, + + /** + * This error indicates, that the program contains an unsupported interaction between different versions of CUDA Dynamic Parallelism. + */ + cudaErrorCdpVersionMismatch = 812, + + /** + * The operation is not permitted when the stream is capturing. + */ + cudaErrorStreamCaptureUnsupported = 900, + + /** + * The current capture sequence on the stream has been invalidated due to + * a previous error. + */ + cudaErrorStreamCaptureInvalidated = 901, + + /** + * The operation would have resulted in a merge of two independent capture + * sequences. + */ + cudaErrorStreamCaptureMerge = 902, + + /** + * The capture was not initiated in this stream. + */ + cudaErrorStreamCaptureUnmatched = 903, + + /** + * The capture sequence contains a fork that was not joined to the primary + * stream. + */ + cudaErrorStreamCaptureUnjoined = 904, + + /** + * A dependency would have been created which crosses the capture sequence + * boundary. Only implicit in-stream ordering dependencies are allowed to + * cross the boundary. + */ + cudaErrorStreamCaptureIsolation = 905, + + /** + * The operation would have resulted in a disallowed implicit dependency on + * a current capture sequence from cudaStreamLegacy. + */ + cudaErrorStreamCaptureImplicit = 906, + + /** + * The operation is not permitted on an event which was last recorded in a + * capturing stream. + */ + cudaErrorCapturedEvent = 907, + + /** + * A stream capture sequence not initiated with the ::cudaStreamCaptureModeRelaxed + * argument to ::cudaStreamBeginCapture was passed to ::cudaStreamEndCapture in a + * different thread. + */ + cudaErrorStreamCaptureWrongThread = 908, + + /** + * This indicates that the wait operation has timed out. + */ + cudaErrorTimeout = 909, + + /** + * This error indicates that the graph update was not performed because it included + * changes which violated constraints specific to instantiated graph update. + */ + cudaErrorGraphExecUpdateFailure = 910, + + /** + * This indicates that an async error has occurred in a device outside of CUDA. + * If CUDA was waiting for an external device's signal before consuming shared data, + * the external device signaled an error indicating that the data is not valid for + * consumption. This leaves the process in an inconsistent state and any further CUDA + * work will return the same error. To continue using CUDA, the process must be + * terminated and relaunched. + */ + cudaErrorExternalDevice = 911, + + /** + * This indicates that a kernel launch error has occurred due to cluster + * misconfiguration. + */ + cudaErrorInvalidClusterSize = 912, + + /** + * This indicates that an unknown internal error has occurred. + */ + cudaErrorUnknown = 999 + + /** + * Any unhandled CUDA driver error is added to this value and returned via + * the runtime. Production releases of CUDA should not return such errors. + * \deprecated + * This error return is deprecated as of CUDA 4.1. + */ + , cudaErrorApiFailureBase = 10000 +}; + +/** + * Channel format kind + */ +enum __device_builtin__ cudaChannelFormatKind +{ + cudaChannelFormatKindSigned = 0, /**< Signed channel format */ + cudaChannelFormatKindUnsigned = 1, /**< Unsigned channel format */ + cudaChannelFormatKindFloat = 2, /**< Float channel format */ + cudaChannelFormatKindNone = 3, /**< No channel format */ + cudaChannelFormatKindNV12 = 4, /**< Unsigned 8-bit integers, planar 4:2:0 YUV format */ + cudaChannelFormatKindUnsignedNormalized8X1 = 5, /**< 1 channel unsigned 8-bit normalized integer */ + cudaChannelFormatKindUnsignedNormalized8X2 = 6, /**< 2 channel unsigned 8-bit normalized integer */ + cudaChannelFormatKindUnsignedNormalized8X4 = 7, /**< 4 channel unsigned 8-bit normalized integer */ + cudaChannelFormatKindUnsignedNormalized16X1 = 8, /**< 1 channel unsigned 16-bit normalized integer */ + cudaChannelFormatKindUnsignedNormalized16X2 = 9, /**< 2 channel unsigned 16-bit normalized integer */ + cudaChannelFormatKindUnsignedNormalized16X4 = 10, /**< 4 channel unsigned 16-bit normalized integer */ + cudaChannelFormatKindSignedNormalized8X1 = 11, /**< 1 channel signed 8-bit normalized integer */ + cudaChannelFormatKindSignedNormalized8X2 = 12, /**< 2 channel signed 8-bit normalized integer */ + cudaChannelFormatKindSignedNormalized8X4 = 13, /**< 4 channel signed 8-bit normalized integer */ + cudaChannelFormatKindSignedNormalized16X1 = 14, /**< 1 channel signed 16-bit normalized integer */ + cudaChannelFormatKindSignedNormalized16X2 = 15, /**< 2 channel signed 16-bit normalized integer */ + cudaChannelFormatKindSignedNormalized16X4 = 16, /**< 4 channel signed 16-bit normalized integer */ + cudaChannelFormatKindUnsignedBlockCompressed1 = 17, /**< 4 channel unsigned normalized block-compressed (BC1 compression) format */ + cudaChannelFormatKindUnsignedBlockCompressed1SRGB = 18, /**< 4 channel unsigned normalized block-compressed (BC1 compression) format with sRGB encoding*/ + cudaChannelFormatKindUnsignedBlockCompressed2 = 19, /**< 4 channel unsigned normalized block-compressed (BC2 compression) format */ + cudaChannelFormatKindUnsignedBlockCompressed2SRGB = 20, /**< 4 channel unsigned normalized block-compressed (BC2 compression) format with sRGB encoding */ + cudaChannelFormatKindUnsignedBlockCompressed3 = 21, /**< 4 channel unsigned normalized block-compressed (BC3 compression) format */ + cudaChannelFormatKindUnsignedBlockCompressed3SRGB = 22, /**< 4 channel unsigned normalized block-compressed (BC3 compression) format with sRGB encoding */ + cudaChannelFormatKindUnsignedBlockCompressed4 = 23, /**< 1 channel unsigned normalized block-compressed (BC4 compression) format */ + cudaChannelFormatKindSignedBlockCompressed4 = 24, /**< 1 channel signed normalized block-compressed (BC4 compression) format */ + cudaChannelFormatKindUnsignedBlockCompressed5 = 25, /**< 2 channel unsigned normalized block-compressed (BC5 compression) format */ + cudaChannelFormatKindSignedBlockCompressed5 = 26, /**< 2 channel signed normalized block-compressed (BC5 compression) format */ + cudaChannelFormatKindUnsignedBlockCompressed6H = 27, /**< 3 channel unsigned half-float block-compressed (BC6H compression) format */ + cudaChannelFormatKindSignedBlockCompressed6H = 28, /**< 3 channel signed half-float block-compressed (BC6H compression) format */ + cudaChannelFormatKindUnsignedBlockCompressed7 = 29, /**< 4 channel unsigned normalized block-compressed (BC7 compression) format */ + cudaChannelFormatKindUnsignedBlockCompressed7SRGB = 30 /**< 4 channel unsigned normalized block-compressed (BC7 compression) format with sRGB encoding */ +}; + +/** + * CUDA Channel format descriptor + */ +struct __device_builtin__ cudaChannelFormatDesc +{ + int x; /**< x */ + int y; /**< y */ + int z; /**< z */ + int w; /**< w */ + enum cudaChannelFormatKind f; /**< Channel format kind */ +}; + +/** + * CUDA array + */ +typedef struct cudaArray *cudaArray_t; + +/** + * CUDA array (as source copy argument) + */ +typedef const struct cudaArray *cudaArray_const_t; + +struct cudaArray; + +/** + * CUDA mipmapped array + */ +typedef struct cudaMipmappedArray *cudaMipmappedArray_t; + +/** + * CUDA mipmapped array (as source argument) + */ +typedef const struct cudaMipmappedArray *cudaMipmappedArray_const_t; + +struct cudaMipmappedArray; + +/** + * Indicates that the layered sparse CUDA array or CUDA mipmapped array has a single mip tail region for all layers + */ +#define cudaArraySparsePropertiesSingleMipTail 0x1 + +/** + * Sparse CUDA array and CUDA mipmapped array properties + */ +struct __device_builtin__ cudaArraySparseProperties { + struct { + unsigned int width; /**< Tile width in elements */ + unsigned int height; /**< Tile height in elements */ + unsigned int depth; /**< Tile depth in elements */ + } tileExtent; + unsigned int miptailFirstLevel; /**< First mip level at which the mip tail begins */ + unsigned long long miptailSize; /**< Total size of the mip tail. */ + unsigned int flags; /**< Flags will either be zero or ::cudaArraySparsePropertiesSingleMipTail */ + unsigned int reserved[4]; +}; + +/** + * CUDA array and CUDA mipmapped array memory requirements + */ +struct __device_builtin__ cudaArrayMemoryRequirements { + size_t size; /**< Total size of the array. */ + size_t alignment; /**< Alignment necessary for mapping the array. */ + unsigned int reserved[4]; +}; + +/** + * CUDA memory types + */ +enum __device_builtin__ cudaMemoryType +{ + cudaMemoryTypeUnregistered = 0, /**< Unregistered memory */ + cudaMemoryTypeHost = 1, /**< Host memory */ + cudaMemoryTypeDevice = 2, /**< Device memory */ + cudaMemoryTypeManaged = 3 /**< Managed memory */ +}; + +/** + * CUDA memory copy types + */ +enum __device_builtin__ cudaMemcpyKind +{ + cudaMemcpyHostToHost = 0, /**< Host -> Host */ + cudaMemcpyHostToDevice = 1, /**< Host -> Device */ + cudaMemcpyDeviceToHost = 2, /**< Device -> Host */ + cudaMemcpyDeviceToDevice = 3, /**< Device -> Device */ + cudaMemcpyDefault = 4 /**< Direction of the transfer is inferred from the pointer values. Requires unified virtual addressing */ +}; + +/** + * CUDA Pitched memory pointer + * + * \sa ::make_cudaPitchedPtr + */ +struct __device_builtin__ cudaPitchedPtr +{ + void *ptr; /**< Pointer to allocated memory */ + size_t pitch; /**< Pitch of allocated memory in bytes */ + size_t xsize; /**< Logical width of allocation in elements */ + size_t ysize; /**< Logical height of allocation in elements */ +}; + +/** + * CUDA extent + * + * \sa ::make_cudaExtent + */ +struct __device_builtin__ cudaExtent +{ + size_t width; /**< Width in elements when referring to array memory, in bytes when referring to linear memory */ + size_t height; /**< Height in elements */ + size_t depth; /**< Depth in elements */ +}; + +/** + * CUDA 3D position + * + * \sa ::make_cudaPos + */ +struct __device_builtin__ cudaPos +{ + size_t x; /**< x */ + size_t y; /**< y */ + size_t z; /**< z */ +}; + +/** + * CUDA 3D memory copying parameters + */ +struct __device_builtin__ cudaMemcpy3DParms +{ + cudaArray_t srcArray; /**< Source memory address */ + struct cudaPos srcPos; /**< Source position offset */ + struct cudaPitchedPtr srcPtr; /**< Pitched source memory address */ + + cudaArray_t dstArray; /**< Destination memory address */ + struct cudaPos dstPos; /**< Destination position offset */ + struct cudaPitchedPtr dstPtr; /**< Pitched destination memory address */ + + struct cudaExtent extent; /**< Requested memory copy size */ + enum cudaMemcpyKind kind; /**< Type of transfer */ +}; + +/** + * Memcpy node parameters + */ +struct __device_builtin__ cudaMemcpyNodeParams { + int flags; /**< Must be zero */ + int reserved[3]; /**< Must be zero */ + struct cudaMemcpy3DParms copyParams; /**< Parameters for the memory copy */ +}; + +/** + * CUDA 3D cross-device memory copying parameters + */ +struct __device_builtin__ cudaMemcpy3DPeerParms +{ + cudaArray_t srcArray; /**< Source memory address */ + struct cudaPos srcPos; /**< Source position offset */ + struct cudaPitchedPtr srcPtr; /**< Pitched source memory address */ + int srcDevice; /**< Source device */ + + cudaArray_t dstArray; /**< Destination memory address */ + struct cudaPos dstPos; /**< Destination position offset */ + struct cudaPitchedPtr dstPtr; /**< Pitched destination memory address */ + int dstDevice; /**< Destination device */ + + struct cudaExtent extent; /**< Requested memory copy size */ +}; + +/** + * CUDA Memset node parameters + */ +struct __device_builtin__ cudaMemsetParams { + void *dst; /**< Destination device pointer */ + size_t pitch; /**< Pitch of destination device pointer. Unused if height is 1 */ + unsigned int value; /**< Value to be set */ + unsigned int elementSize; /**< Size of each element in bytes. Must be 1, 2, or 4. */ + size_t width; /**< Width of the row in elements */ + size_t height; /**< Number of rows */ +}; + +/** + * CUDA Memset node parameters + */ +struct __device_builtin__ cudaMemsetParamsV2 { + void *dst; /**< Destination device pointer */ + size_t pitch; /**< Pitch of destination device pointer. Unused if height is 1 */ + unsigned int value; /**< Value to be set */ + unsigned int elementSize; /**< Size of each element in bytes. Must be 1, 2, or 4. */ + size_t width; /**< Width of the row in elements */ + size_t height; /**< Number of rows */ +}; + +/** + * Specifies performance hint with ::cudaAccessPolicyWindow for hitProp and missProp members. + */ +enum __device_builtin__ cudaAccessProperty { + cudaAccessPropertyNormal = 0, /**< Normal cache persistence. */ + cudaAccessPropertyStreaming = 1, /**< Streaming access is less likely to persit from cache. */ + cudaAccessPropertyPersisting = 2 /**< Persisting access is more likely to persist in cache.*/ +}; + +/** + * Specifies an access policy for a window, a contiguous extent of memory + * beginning at base_ptr and ending at base_ptr + num_bytes. + * Partition into many segments and assign segments such that. + * sum of "hit segments" / window == approx. ratio. + * sum of "miss segments" / window == approx 1-ratio. + * Segments and ratio specifications are fitted to the capabilities of + * the architecture. + * Accesses in a hit segment apply the hitProp access policy. + * Accesses in a miss segment apply the missProp access policy. + */ +struct __device_builtin__ cudaAccessPolicyWindow { + void *base_ptr; /**< Starting address of the access policy window. CUDA driver may align it. */ + size_t num_bytes; /**< Size in bytes of the window policy. CUDA driver may restrict the maximum size and alignment. */ + float hitRatio; /**< hitRatio specifies percentage of lines assigned hitProp, rest are assigned missProp. */ + enum cudaAccessProperty hitProp; /**< ::CUaccessProperty set for hit. */ + enum cudaAccessProperty missProp; /**< ::CUaccessProperty set for miss. Must be either NORMAL or STREAMING. */ +}; + +#ifdef _WIN32 +#define CUDART_CB __stdcall +#else +#define CUDART_CB +#endif + +/** + * CUDA host function + * \param userData Argument value passed to the function + */ +typedef void (CUDART_CB *cudaHostFn_t)(void *userData); + +/** + * CUDA host node parameters + */ +struct __device_builtin__ cudaHostNodeParams { + cudaHostFn_t fn; /**< The function to call when the node executes */ + void* userData; /**< Argument to pass to the function */ +}; + +/** + * CUDA host node parameters + */ +struct __device_builtin__ cudaHostNodeParamsV2 { + cudaHostFn_t fn; /**< The function to call when the node executes */ + void* userData; /**< Argument to pass to the function */ +}; + +/** + * Possible stream capture statuses returned by ::cudaStreamIsCapturing + */ +enum __device_builtin__ cudaStreamCaptureStatus { + cudaStreamCaptureStatusNone = 0, /**< Stream is not capturing */ + cudaStreamCaptureStatusActive = 1, /**< Stream is actively capturing */ + cudaStreamCaptureStatusInvalidated = 2 /**< Stream is part of a capture sequence that + has been invalidated, but not terminated */ +}; + +/** + * Possible modes for stream capture thread interactions. For more details see + * ::cudaStreamBeginCapture and ::cudaThreadExchangeStreamCaptureMode + */ +enum __device_builtin__ cudaStreamCaptureMode { + cudaStreamCaptureModeGlobal = 0, + cudaStreamCaptureModeThreadLocal = 1, + cudaStreamCaptureModeRelaxed = 2 +}; + +enum __device_builtin__ cudaSynchronizationPolicy { + cudaSyncPolicyAuto = 1, + cudaSyncPolicySpin = 2, + cudaSyncPolicyYield = 3, + cudaSyncPolicyBlockingSync = 4 +}; + +/** + * Cluster scheduling policies. These may be passed to ::cudaFuncSetAttribute + */ +enum __device_builtin__ cudaClusterSchedulingPolicy { + cudaClusterSchedulingPolicyDefault = 0, /**< the default policy */ + cudaClusterSchedulingPolicySpread = 1, /**< spread the blocks within a cluster to the SMs */ + cudaClusterSchedulingPolicyLoadBalancing = 2 /**< allow the hardware to load-balance the blocks in a cluster to the SMs */ +}; + +/** + * Flags for ::cudaStreamUpdateCaptureDependencies + */ +enum __device_builtin__ cudaStreamUpdateCaptureDependenciesFlags { + cudaStreamAddCaptureDependencies = 0x0, /**< Add new nodes to the dependency set */ + cudaStreamSetCaptureDependencies = 0x1 /**< Replace the dependency set with the new nodes */ +}; + +/** + * Flags for user objects for graphs + */ +enum __device_builtin__ cudaUserObjectFlags { + cudaUserObjectNoDestructorSync = 0x1 /**< Indicates the destructor execution is not synchronized by any CUDA handle. */ +}; + +/** + * Flags for retaining user object references for graphs + */ +enum __device_builtin__ cudaUserObjectRetainFlags { + cudaGraphUserObjectMove = 0x1 /**< Transfer references from the caller rather than creating new references. */ +}; + +/** + * CUDA graphics interop resource + */ +struct cudaGraphicsResource; + +/** + * CUDA graphics interop register flags + */ +enum __device_builtin__ cudaGraphicsRegisterFlags +{ + cudaGraphicsRegisterFlagsNone = 0, /**< Default */ + cudaGraphicsRegisterFlagsReadOnly = 1, /**< CUDA will not write to this resource */ + cudaGraphicsRegisterFlagsWriteDiscard = 2, /**< CUDA will only write to and will not read from this resource */ + cudaGraphicsRegisterFlagsSurfaceLoadStore = 4, /**< CUDA will bind this resource to a surface reference */ + cudaGraphicsRegisterFlagsTextureGather = 8 /**< CUDA will perform texture gather operations on this resource */ +}; + +/** + * CUDA graphics interop map flags + */ +enum __device_builtin__ cudaGraphicsMapFlags +{ + cudaGraphicsMapFlagsNone = 0, /**< Default; Assume resource can be read/written */ + cudaGraphicsMapFlagsReadOnly = 1, /**< CUDA will not write to this resource */ + cudaGraphicsMapFlagsWriteDiscard = 2 /**< CUDA will only write to and will not read from this resource */ +}; + +/** + * CUDA graphics interop array indices for cube maps + */ +enum __device_builtin__ cudaGraphicsCubeFace +{ + cudaGraphicsCubeFacePositiveX = 0x00, /**< Positive X face of cubemap */ + cudaGraphicsCubeFaceNegativeX = 0x01, /**< Negative X face of cubemap */ + cudaGraphicsCubeFacePositiveY = 0x02, /**< Positive Y face of cubemap */ + cudaGraphicsCubeFaceNegativeY = 0x03, /**< Negative Y face of cubemap */ + cudaGraphicsCubeFacePositiveZ = 0x04, /**< Positive Z face of cubemap */ + cudaGraphicsCubeFaceNegativeZ = 0x05 /**< Negative Z face of cubemap */ +}; + +/** + * CUDA resource types + */ +enum __device_builtin__ cudaResourceType +{ + cudaResourceTypeArray = 0x00, /**< Array resource */ + cudaResourceTypeMipmappedArray = 0x01, /**< Mipmapped array resource */ + cudaResourceTypeLinear = 0x02, /**< Linear resource */ + cudaResourceTypePitch2D = 0x03 /**< Pitch 2D resource */ +}; + +/** + * CUDA texture resource view formats + */ +enum __device_builtin__ cudaResourceViewFormat +{ + cudaResViewFormatNone = 0x00, /**< No resource view format (use underlying resource format) */ + cudaResViewFormatUnsignedChar1 = 0x01, /**< 1 channel unsigned 8-bit integers */ + cudaResViewFormatUnsignedChar2 = 0x02, /**< 2 channel unsigned 8-bit integers */ + cudaResViewFormatUnsignedChar4 = 0x03, /**< 4 channel unsigned 8-bit integers */ + cudaResViewFormatSignedChar1 = 0x04, /**< 1 channel signed 8-bit integers */ + cudaResViewFormatSignedChar2 = 0x05, /**< 2 channel signed 8-bit integers */ + cudaResViewFormatSignedChar4 = 0x06, /**< 4 channel signed 8-bit integers */ + cudaResViewFormatUnsignedShort1 = 0x07, /**< 1 channel unsigned 16-bit integers */ + cudaResViewFormatUnsignedShort2 = 0x08, /**< 2 channel unsigned 16-bit integers */ + cudaResViewFormatUnsignedShort4 = 0x09, /**< 4 channel unsigned 16-bit integers */ + cudaResViewFormatSignedShort1 = 0x0a, /**< 1 channel signed 16-bit integers */ + cudaResViewFormatSignedShort2 = 0x0b, /**< 2 channel signed 16-bit integers */ + cudaResViewFormatSignedShort4 = 0x0c, /**< 4 channel signed 16-bit integers */ + cudaResViewFormatUnsignedInt1 = 0x0d, /**< 1 channel unsigned 32-bit integers */ + cudaResViewFormatUnsignedInt2 = 0x0e, /**< 2 channel unsigned 32-bit integers */ + cudaResViewFormatUnsignedInt4 = 0x0f, /**< 4 channel unsigned 32-bit integers */ + cudaResViewFormatSignedInt1 = 0x10, /**< 1 channel signed 32-bit integers */ + cudaResViewFormatSignedInt2 = 0x11, /**< 2 channel signed 32-bit integers */ + cudaResViewFormatSignedInt4 = 0x12, /**< 4 channel signed 32-bit integers */ + cudaResViewFormatHalf1 = 0x13, /**< 1 channel 16-bit floating point */ + cudaResViewFormatHalf2 = 0x14, /**< 2 channel 16-bit floating point */ + cudaResViewFormatHalf4 = 0x15, /**< 4 channel 16-bit floating point */ + cudaResViewFormatFloat1 = 0x16, /**< 1 channel 32-bit floating point */ + cudaResViewFormatFloat2 = 0x17, /**< 2 channel 32-bit floating point */ + cudaResViewFormatFloat4 = 0x18, /**< 4 channel 32-bit floating point */ + cudaResViewFormatUnsignedBlockCompressed1 = 0x19, /**< Block compressed 1 */ + cudaResViewFormatUnsignedBlockCompressed2 = 0x1a, /**< Block compressed 2 */ + cudaResViewFormatUnsignedBlockCompressed3 = 0x1b, /**< Block compressed 3 */ + cudaResViewFormatUnsignedBlockCompressed4 = 0x1c, /**< Block compressed 4 unsigned */ + cudaResViewFormatSignedBlockCompressed4 = 0x1d, /**< Block compressed 4 signed */ + cudaResViewFormatUnsignedBlockCompressed5 = 0x1e, /**< Block compressed 5 unsigned */ + cudaResViewFormatSignedBlockCompressed5 = 0x1f, /**< Block compressed 5 signed */ + cudaResViewFormatUnsignedBlockCompressed6H = 0x20, /**< Block compressed 6 unsigned half-float */ + cudaResViewFormatSignedBlockCompressed6H = 0x21, /**< Block compressed 6 signed half-float */ + cudaResViewFormatUnsignedBlockCompressed7 = 0x22 /**< Block compressed 7 */ +}; + +/** + * CUDA resource descriptor + */ +struct __device_builtin__ cudaResourceDesc { + enum cudaResourceType resType; /**< Resource type */ + + union { + struct { + cudaArray_t array; /**< CUDA array */ + } array; + struct { + cudaMipmappedArray_t mipmap; /**< CUDA mipmapped array */ + } mipmap; + struct { + void *devPtr; /**< Device pointer */ + struct cudaChannelFormatDesc desc; /**< Channel descriptor */ + size_t sizeInBytes; /**< Size in bytes */ + } linear; + struct { + void *devPtr; /**< Device pointer */ + struct cudaChannelFormatDesc desc; /**< Channel descriptor */ + size_t width; /**< Width of the array in elements */ + size_t height; /**< Height of the array in elements */ + size_t pitchInBytes; /**< Pitch between two rows in bytes */ + } pitch2D; + } res; +}; + +/** + * CUDA resource view descriptor + */ +struct __device_builtin__ cudaResourceViewDesc +{ + enum cudaResourceViewFormat format; /**< Resource view format */ + size_t width; /**< Width of the resource view */ + size_t height; /**< Height of the resource view */ + size_t depth; /**< Depth of the resource view */ + unsigned int firstMipmapLevel; /**< First defined mipmap level */ + unsigned int lastMipmapLevel; /**< Last defined mipmap level */ + unsigned int firstLayer; /**< First layer index */ + unsigned int lastLayer; /**< Last layer index */ +}; + +/** + * CUDA pointer attributes + */ +struct __device_builtin__ cudaPointerAttributes +{ + /** + * The type of memory - ::cudaMemoryTypeUnregistered, ::cudaMemoryTypeHost, + * ::cudaMemoryTypeDevice or ::cudaMemoryTypeManaged. + */ + enum cudaMemoryType type; + + /** + * The device against which the memory was allocated or registered. + * If the memory type is ::cudaMemoryTypeDevice then this identifies + * the device on which the memory referred physically resides. If + * the memory type is ::cudaMemoryTypeHost or::cudaMemoryTypeManaged then + * this identifies the device which was current when the memory was allocated + * or registered (and if that device is deinitialized then this allocation + * will vanish with that device's state). + */ + int device; + + /** + * The address which may be dereferenced on the current device to access + * the memory or NULL if no such address exists. + */ + void *devicePointer; + + /** + * The address which may be dereferenced on the host to access the + * memory or NULL if no such address exists. + * + * \note CUDA doesn't check if unregistered memory is allocated so this field + * may contain invalid pointer if an invalid pointer has been passed to CUDA. + */ + void *hostPointer; +}; + +/** + * CUDA function attributes + */ +struct __device_builtin__ cudaFuncAttributes +{ + /** + * The size in bytes of statically-allocated shared memory per block + * required by this function. This does not include dynamically-allocated + * shared memory requested by the user at runtime. + */ + size_t sharedSizeBytes; + + /** + * The size in bytes of user-allocated constant memory required by this + * function. + */ + size_t constSizeBytes; + + /** + * The size in bytes of local memory used by each thread of this function. + */ + size_t localSizeBytes; + + /** + * The maximum number of threads per block, beyond which a launch of the + * function would fail. This number depends on both the function and the + * device on which the function is currently loaded. + */ + int maxThreadsPerBlock; + + /** + * The number of registers used by each thread of this function. + */ + int numRegs; + + /** + * The PTX virtual architecture version for which the function was + * compiled. This value is the major PTX version * 10 + the minor PTX + * version, so a PTX version 1.3 function would return the value 13. + */ + int ptxVersion; + + /** + * The binary architecture version for which the function was compiled. + * This value is the major binary version * 10 + the minor binary version, + * so a binary version 1.3 function would return the value 13. + */ + int binaryVersion; + + /** + * The attribute to indicate whether the function has been compiled with + * user specified option "-Xptxas --dlcm=ca" set. + */ + int cacheModeCA; + + /** + * The maximum size in bytes of dynamic shared memory per block for + * this function. Any launch must have a dynamic shared memory size + * smaller than this value. + */ + int maxDynamicSharedSizeBytes; + + /** + * On devices where the L1 cache and shared memory use the same hardware resources, + * this sets the shared memory carveout preference, in percent of the maximum shared memory. + * Refer to ::cudaDevAttrMaxSharedMemoryPerMultiprocessor. + * This is only a hint, and the driver can choose a different ratio if required to execute the function. + * See ::cudaFuncSetAttribute + */ + int preferredShmemCarveout; + + /** + * If this attribute is set, the kernel must launch with a valid cluster dimension + * specified. + */ + int clusterDimMustBeSet; + + /** + * The required cluster width/height/depth in blocks. The values must either + * all be 0 or all be positive. The validity of the cluster dimensions is + * otherwise checked at launch time. + * + * If the value is set during compile time, it cannot be set at runtime. + * Setting it at runtime should return cudaErrorNotPermitted. + * See ::cudaFuncSetAttribute + */ + int requiredClusterWidth; + int requiredClusterHeight; + int requiredClusterDepth; + + /** + * The block scheduling policy of a function. + * See ::cudaFuncSetAttribute + */ + int clusterSchedulingPolicyPreference; + + /** + * Whether the function can be launched with non-portable cluster size. 1 is + * allowed, 0 is disallowed. A non-portable cluster size may only function + * on the specific SKUs the program is tested on. The launch might fail if + * the program is run on a different hardware platform. + * + * CUDA API provides ::cudaOccupancyMaxActiveClusters to assist with checking + * whether the desired size can be launched on the current device. + * + * Portable Cluster Size + * + * A portable cluster size is guaranteed to be functional on all compute + * capabilities higher than the target compute capability. The portable + * cluster size for sm_90 is 8 blocks per cluster. This value may increase + * for future compute capabilities. + * + * The specific hardware unit may support higher cluster sizes that’s not + * guaranteed to be portable. + * See ::cudaFuncSetAttribute + */ + int nonPortableClusterSizeAllowed; + + /** + * Reserved for future use. + */ + int reserved[16]; +}; + +/** + * CUDA function attributes that can be set using ::cudaFuncSetAttribute + */ +enum __device_builtin__ cudaFuncAttribute +{ + cudaFuncAttributeMaxDynamicSharedMemorySize = 8, /**< Maximum dynamic shared memory size */ + cudaFuncAttributePreferredSharedMemoryCarveout = 9, /**< Preferred shared memory-L1 cache split */ + cudaFuncAttributeClusterDimMustBeSet = 10, /**< Indicator to enforce valid cluster dimension specification on kernel launch */ + cudaFuncAttributeRequiredClusterWidth = 11, /**< Required cluster width */ + cudaFuncAttributeRequiredClusterHeight = 12, /**< Required cluster height */ + cudaFuncAttributeRequiredClusterDepth = 13, /**< Required cluster depth */ + cudaFuncAttributeNonPortableClusterSizeAllowed = 14, /**< Whether non-portable cluster scheduling policy is supported */ + cudaFuncAttributeClusterSchedulingPolicyPreference = 15, /**< Required cluster scheduling policy preference */ + cudaFuncAttributeMax +}; + +/** + * CUDA function cache configurations + */ +enum __device_builtin__ cudaFuncCache +{ + cudaFuncCachePreferNone = 0, /**< Default function cache configuration, no preference */ + cudaFuncCachePreferShared = 1, /**< Prefer larger shared memory and smaller L1 cache */ + cudaFuncCachePreferL1 = 2, /**< Prefer larger L1 cache and smaller shared memory */ + cudaFuncCachePreferEqual = 3 /**< Prefer equal size L1 cache and shared memory */ +}; + +/** + * CUDA shared memory configuration + * \deprecated + */ +enum __device_builtin__ cudaSharedMemConfig +{ + cudaSharedMemBankSizeDefault = 0, + cudaSharedMemBankSizeFourByte = 1, + cudaSharedMemBankSizeEightByte = 2 +}; + +/** + * Shared memory carveout configurations. These may be passed to cudaFuncSetAttribute + */ +enum __device_builtin__ cudaSharedCarveout { + cudaSharedmemCarveoutDefault = -1, /**< No preference for shared memory or L1 (default) */ + cudaSharedmemCarveoutMaxShared = 100, /**< Prefer maximum available shared memory, minimum L1 cache */ + cudaSharedmemCarveoutMaxL1 = 0 /**< Prefer maximum available L1 cache, minimum shared memory */ +}; + +/** + * CUDA device compute modes + */ +enum __device_builtin__ cudaComputeMode +{ + cudaComputeModeDefault = 0, /**< Default compute mode (Multiple threads can use ::cudaSetDevice() with this device) */ + cudaComputeModeExclusive = 1, /**< Compute-exclusive-thread mode (Only one thread in one process will be able to use ::cudaSetDevice() with this device) */ + cudaComputeModeProhibited = 2, /**< Compute-prohibited mode (No threads can use ::cudaSetDevice() with this device) */ + cudaComputeModeExclusiveProcess = 3 /**< Compute-exclusive-process mode (Many threads in one process will be able to use ::cudaSetDevice() with this device) */ +}; + +/** + * CUDA Limits + */ +enum __device_builtin__ cudaLimit +{ + cudaLimitStackSize = 0x00, /**< GPU thread stack size */ + cudaLimitPrintfFifoSize = 0x01, /**< GPU printf FIFO size */ + cudaLimitMallocHeapSize = 0x02, /**< GPU malloc heap size */ + cudaLimitDevRuntimeSyncDepth = 0x03, /**< GPU device runtime synchronize depth */ + cudaLimitDevRuntimePendingLaunchCount = 0x04, /**< GPU device runtime pending launch count */ + cudaLimitMaxL2FetchGranularity = 0x05, /**< A value between 0 and 128 that indicates the maximum fetch granularity of L2 (in Bytes). This is a hint */ + cudaLimitPersistingL2CacheSize = 0x06 /**< A size in bytes for L2 persisting lines cache size */ +}; + +/** + * CUDA Memory Advise values + */ +enum __device_builtin__ cudaMemoryAdvise +{ + cudaMemAdviseSetReadMostly = 1, /**< Data will mostly be read and only occassionally be written to */ + cudaMemAdviseUnsetReadMostly = 2, /**< Undo the effect of ::cudaMemAdviseSetReadMostly */ + cudaMemAdviseSetPreferredLocation = 3, /**< Set the preferred location for the data as the specified device */ + cudaMemAdviseUnsetPreferredLocation = 4, /**< Clear the preferred location for the data */ + cudaMemAdviseSetAccessedBy = 5, /**< Data will be accessed by the specified device, so prevent page faults as much as possible */ + cudaMemAdviseUnsetAccessedBy = 6 /**< Let the Unified Memory subsystem decide on the page faulting policy for the specified device */ +}; + +/** + * CUDA range attributes + */ +enum __device_builtin__ cudaMemRangeAttribute +{ + cudaMemRangeAttributeReadMostly = 1, /**< Whether the range will mostly be read and only occassionally be written to */ + cudaMemRangeAttributePreferredLocation = 2, /**< The preferred location of the range */ + cudaMemRangeAttributeAccessedBy = 3, /**< Memory range has ::cudaMemAdviseSetAccessedBy set for specified device */ + cudaMemRangeAttributeLastPrefetchLocation = 4 /**< The last location to which the range was prefetched */ + , cudaMemRangeAttributePreferredLocationType = 5 /**< The preferred location type of the range */ + , cudaMemRangeAttributePreferredLocationId = 6 /**< The preferred location id of the range */ + , cudaMemRangeAttributeLastPrefetchLocationType = 7 /**< The last location type to which the range was prefetched */ + , cudaMemRangeAttributeLastPrefetchLocationId = 8 /**< The last location id to which the range was prefetched */ +}; + +/** + * CUDA GPUDirect RDMA flush writes APIs supported on the device + */ +enum __device_builtin__ cudaFlushGPUDirectRDMAWritesOptions { + cudaFlushGPUDirectRDMAWritesOptionHost = 1<<0, /**< ::cudaDeviceFlushGPUDirectRDMAWrites() and its CUDA Driver API counterpart are supported on the device. */ + cudaFlushGPUDirectRDMAWritesOptionMemOps = 1<<1 /**< The ::CU_STREAM_WAIT_VALUE_FLUSH flag and the ::CU_STREAM_MEM_OP_FLUSH_REMOTE_WRITES MemOp are supported on the CUDA device. */ +}; + +/** + * CUDA GPUDirect RDMA flush writes ordering features of the device + */ +enum __device_builtin__ cudaGPUDirectRDMAWritesOrdering { + cudaGPUDirectRDMAWritesOrderingNone = 0, /**< The device does not natively support ordering of GPUDirect RDMA writes. ::cudaFlushGPUDirectRDMAWrites() can be leveraged if supported. */ + cudaGPUDirectRDMAWritesOrderingOwner = 100, /**< Natively, the device can consistently consume GPUDirect RDMA writes, although other CUDA devices may not. */ + cudaGPUDirectRDMAWritesOrderingAllDevices = 200 /**< Any CUDA device in the system can consistently consume GPUDirect RDMA writes to this device. */ +}; + +/** + * CUDA GPUDirect RDMA flush writes scopes + */ +enum __device_builtin__ cudaFlushGPUDirectRDMAWritesScope { + cudaFlushGPUDirectRDMAWritesToOwner = 100, /**< Blocks until remote writes are visible to the CUDA device context owning the data. */ + cudaFlushGPUDirectRDMAWritesToAllDevices = 200 /**< Blocks until remote writes are visible to all CUDA device contexts. */ +}; + +/** + * CUDA GPUDirect RDMA flush writes targets + */ +enum __device_builtin__ cudaFlushGPUDirectRDMAWritesTarget { + cudaFlushGPUDirectRDMAWritesTargetCurrentDevice /**< Sets the target for ::cudaDeviceFlushGPUDirectRDMAWrites() to the currently active CUDA device context. */ +}; + + +/** + * CUDA device attributes + */ +enum __device_builtin__ cudaDeviceAttr +{ + cudaDevAttrMaxThreadsPerBlock = 1, /**< Maximum number of threads per block */ + cudaDevAttrMaxBlockDimX = 2, /**< Maximum block dimension X */ + cudaDevAttrMaxBlockDimY = 3, /**< Maximum block dimension Y */ + cudaDevAttrMaxBlockDimZ = 4, /**< Maximum block dimension Z */ + cudaDevAttrMaxGridDimX = 5, /**< Maximum grid dimension X */ + cudaDevAttrMaxGridDimY = 6, /**< Maximum grid dimension Y */ + cudaDevAttrMaxGridDimZ = 7, /**< Maximum grid dimension Z */ + cudaDevAttrMaxSharedMemoryPerBlock = 8, /**< Maximum shared memory available per block in bytes */ + cudaDevAttrTotalConstantMemory = 9, /**< Memory available on device for __constant__ variables in a CUDA C kernel in bytes */ + cudaDevAttrWarpSize = 10, /**< Warp size in threads */ + cudaDevAttrMaxPitch = 11, /**< Maximum pitch in bytes allowed by memory copies */ + cudaDevAttrMaxRegistersPerBlock = 12, /**< Maximum number of 32-bit registers available per block */ + cudaDevAttrClockRate = 13, /**< Peak clock frequency in kilohertz */ + cudaDevAttrTextureAlignment = 14, /**< Alignment requirement for textures */ + cudaDevAttrGpuOverlap = 15, /**< Device can possibly copy memory and execute a kernel concurrently */ + cudaDevAttrMultiProcessorCount = 16, /**< Number of multiprocessors on device */ + cudaDevAttrKernelExecTimeout = 17, /**< Specifies whether there is a run time limit on kernels */ + cudaDevAttrIntegrated = 18, /**< Device is integrated with host memory */ + cudaDevAttrCanMapHostMemory = 19, /**< Device can map host memory into CUDA address space */ + cudaDevAttrComputeMode = 20, /**< Compute mode (See ::cudaComputeMode for details) */ + cudaDevAttrMaxTexture1DWidth = 21, /**< Maximum 1D texture width */ + cudaDevAttrMaxTexture2DWidth = 22, /**< Maximum 2D texture width */ + cudaDevAttrMaxTexture2DHeight = 23, /**< Maximum 2D texture height */ + cudaDevAttrMaxTexture3DWidth = 24, /**< Maximum 3D texture width */ + cudaDevAttrMaxTexture3DHeight = 25, /**< Maximum 3D texture height */ + cudaDevAttrMaxTexture3DDepth = 26, /**< Maximum 3D texture depth */ + cudaDevAttrMaxTexture2DLayeredWidth = 27, /**< Maximum 2D layered texture width */ + cudaDevAttrMaxTexture2DLayeredHeight = 28, /**< Maximum 2D layered texture height */ + cudaDevAttrMaxTexture2DLayeredLayers = 29, /**< Maximum layers in a 2D layered texture */ + cudaDevAttrSurfaceAlignment = 30, /**< Alignment requirement for surfaces */ + cudaDevAttrConcurrentKernels = 31, /**< Device can possibly execute multiple kernels concurrently */ + cudaDevAttrEccEnabled = 32, /**< Device has ECC support enabled */ + cudaDevAttrPciBusId = 33, /**< PCI bus ID of the device */ + cudaDevAttrPciDeviceId = 34, /**< PCI device ID of the device */ + cudaDevAttrTccDriver = 35, /**< Device is using TCC driver model */ + cudaDevAttrMemoryClockRate = 36, /**< Peak memory clock frequency in kilohertz */ + cudaDevAttrGlobalMemoryBusWidth = 37, /**< Global memory bus width in bits */ + cudaDevAttrL2CacheSize = 38, /**< Size of L2 cache in bytes */ + cudaDevAttrMaxThreadsPerMultiProcessor = 39, /**< Maximum resident threads per multiprocessor */ + cudaDevAttrAsyncEngineCount = 40, /**< Number of asynchronous engines */ + cudaDevAttrUnifiedAddressing = 41, /**< Device shares a unified address space with the host */ + cudaDevAttrMaxTexture1DLayeredWidth = 42, /**< Maximum 1D layered texture width */ + cudaDevAttrMaxTexture1DLayeredLayers = 43, /**< Maximum layers in a 1D layered texture */ + cudaDevAttrMaxTexture2DGatherWidth = 45, /**< Maximum 2D texture width if cudaArrayTextureGather is set */ + cudaDevAttrMaxTexture2DGatherHeight = 46, /**< Maximum 2D texture height if cudaArrayTextureGather is set */ + cudaDevAttrMaxTexture3DWidthAlt = 47, /**< Alternate maximum 3D texture width */ + cudaDevAttrMaxTexture3DHeightAlt = 48, /**< Alternate maximum 3D texture height */ + cudaDevAttrMaxTexture3DDepthAlt = 49, /**< Alternate maximum 3D texture depth */ + cudaDevAttrPciDomainId = 50, /**< PCI domain ID of the device */ + cudaDevAttrTexturePitchAlignment = 51, /**< Pitch alignment requirement for textures */ + cudaDevAttrMaxTextureCubemapWidth = 52, /**< Maximum cubemap texture width/height */ + cudaDevAttrMaxTextureCubemapLayeredWidth = 53, /**< Maximum cubemap layered texture width/height */ + cudaDevAttrMaxTextureCubemapLayeredLayers = 54, /**< Maximum layers in a cubemap layered texture */ + cudaDevAttrMaxSurface1DWidth = 55, /**< Maximum 1D surface width */ + cudaDevAttrMaxSurface2DWidth = 56, /**< Maximum 2D surface width */ + cudaDevAttrMaxSurface2DHeight = 57, /**< Maximum 2D surface height */ + cudaDevAttrMaxSurface3DWidth = 58, /**< Maximum 3D surface width */ + cudaDevAttrMaxSurface3DHeight = 59, /**< Maximum 3D surface height */ + cudaDevAttrMaxSurface3DDepth = 60, /**< Maximum 3D surface depth */ + cudaDevAttrMaxSurface1DLayeredWidth = 61, /**< Maximum 1D layered surface width */ + cudaDevAttrMaxSurface1DLayeredLayers = 62, /**< Maximum layers in a 1D layered surface */ + cudaDevAttrMaxSurface2DLayeredWidth = 63, /**< Maximum 2D layered surface width */ + cudaDevAttrMaxSurface2DLayeredHeight = 64, /**< Maximum 2D layered surface height */ + cudaDevAttrMaxSurface2DLayeredLayers = 65, /**< Maximum layers in a 2D layered surface */ + cudaDevAttrMaxSurfaceCubemapWidth = 66, /**< Maximum cubemap surface width */ + cudaDevAttrMaxSurfaceCubemapLayeredWidth = 67, /**< Maximum cubemap layered surface width */ + cudaDevAttrMaxSurfaceCubemapLayeredLayers = 68, /**< Maximum layers in a cubemap layered surface */ + cudaDevAttrMaxTexture1DLinearWidth = 69, /**< Maximum 1D linear texture width */ + cudaDevAttrMaxTexture2DLinearWidth = 70, /**< Maximum 2D linear texture width */ + cudaDevAttrMaxTexture2DLinearHeight = 71, /**< Maximum 2D linear texture height */ + cudaDevAttrMaxTexture2DLinearPitch = 72, /**< Maximum 2D linear texture pitch in bytes */ + cudaDevAttrMaxTexture2DMipmappedWidth = 73, /**< Maximum mipmapped 2D texture width */ + cudaDevAttrMaxTexture2DMipmappedHeight = 74, /**< Maximum mipmapped 2D texture height */ + cudaDevAttrComputeCapabilityMajor = 75, /**< Major compute capability version number */ + cudaDevAttrComputeCapabilityMinor = 76, /**< Minor compute capability version number */ + cudaDevAttrMaxTexture1DMipmappedWidth = 77, /**< Maximum mipmapped 1D texture width */ + cudaDevAttrStreamPrioritiesSupported = 78, /**< Device supports stream priorities */ + cudaDevAttrGlobalL1CacheSupported = 79, /**< Device supports caching globals in L1 */ + cudaDevAttrLocalL1CacheSupported = 80, /**< Device supports caching locals in L1 */ + cudaDevAttrMaxSharedMemoryPerMultiprocessor = 81, /**< Maximum shared memory available per multiprocessor in bytes */ + cudaDevAttrMaxRegistersPerMultiprocessor = 82, /**< Maximum number of 32-bit registers available per multiprocessor */ + cudaDevAttrManagedMemory = 83, /**< Device can allocate managed memory on this system */ + cudaDevAttrIsMultiGpuBoard = 84, /**< Device is on a multi-GPU board */ + cudaDevAttrMultiGpuBoardGroupID = 85, /**< Unique identifier for a group of devices on the same multi-GPU board */ + cudaDevAttrHostNativeAtomicSupported = 86, /**< Link between the device and the host supports native atomic operations */ + cudaDevAttrSingleToDoublePrecisionPerfRatio = 87, /**< Ratio of single precision performance (in floating-point operations per second) to double precision performance */ + cudaDevAttrPageableMemoryAccess = 88, /**< Device supports coherently accessing pageable memory without calling cudaHostRegister on it */ + cudaDevAttrConcurrentManagedAccess = 89, /**< Device can coherently access managed memory concurrently with the CPU */ + cudaDevAttrComputePreemptionSupported = 90, /**< Device supports Compute Preemption */ + cudaDevAttrCanUseHostPointerForRegisteredMem = 91, /**< Device can access host registered memory at the same virtual address as the CPU */ + cudaDevAttrReserved92 = 92, + cudaDevAttrReserved93 = 93, + cudaDevAttrReserved94 = 94, + cudaDevAttrCooperativeLaunch = 95, /**< Device supports launching cooperative kernels via ::cudaLaunchCooperativeKernel*/ + cudaDevAttrCooperativeMultiDeviceLaunch = 96, /**< Deprecated, cudaLaunchCooperativeKernelMultiDevice is deprecated. */ + cudaDevAttrMaxSharedMemoryPerBlockOptin = 97, /**< The maximum optin shared memory per block. This value may vary by chip. See ::cudaFuncSetAttribute */ + cudaDevAttrCanFlushRemoteWrites = 98, /**< Device supports flushing of outstanding remote writes. */ + cudaDevAttrHostRegisterSupported = 99, /**< Device supports host memory registration via ::cudaHostRegister. */ + cudaDevAttrPageableMemoryAccessUsesHostPageTables = 100, /**< Device accesses pageable memory via the host's page tables. */ + cudaDevAttrDirectManagedMemAccessFromHost = 101, /**< Host can directly access managed memory on the device without migration. */ + cudaDevAttrMaxBlocksPerMultiprocessor = 106, /**< Maximum number of blocks per multiprocessor */ + cudaDevAttrMaxPersistingL2CacheSize = 108, /**< Maximum L2 persisting lines capacity setting in bytes. */ + cudaDevAttrMaxAccessPolicyWindowSize = 109, /**< Maximum value of cudaAccessPolicyWindow::num_bytes. */ + cudaDevAttrReservedSharedMemoryPerBlock = 111, /**< Shared memory reserved by CUDA driver per block in bytes */ + cudaDevAttrSparseCudaArraySupported = 112, /**< Device supports sparse CUDA arrays and sparse CUDA mipmapped arrays */ + cudaDevAttrHostRegisterReadOnlySupported = 113, /**< Device supports using the ::cudaHostRegister flag cudaHostRegisterReadOnly to register memory that must be mapped as read-only to the GPU */ + cudaDevAttrTimelineSemaphoreInteropSupported = 114, /**< External timeline semaphore interop is supported on the device */ + cudaDevAttrMaxTimelineSemaphoreInteropSupported = 114, /**< Deprecated, External timeline semaphore interop is supported on the device */ + cudaDevAttrMemoryPoolsSupported = 115, /**< Device supports using the ::cudaMallocAsync and ::cudaMemPool family of APIs */ + cudaDevAttrGPUDirectRDMASupported = 116, /**< Device supports GPUDirect RDMA APIs, like nvidia_p2p_get_pages (see https://docs.nvidia.com/cuda/gpudirect-rdma for more information) */ + cudaDevAttrGPUDirectRDMAFlushWritesOptions = 117, /**< The returned attribute shall be interpreted as a bitmask, where the individual bits are listed in the ::cudaFlushGPUDirectRDMAWritesOptions enum */ + cudaDevAttrGPUDirectRDMAWritesOrdering = 118, /**< GPUDirect RDMA writes to the device do not need to be flushed for consumers within the scope indicated by the returned attribute. See ::cudaGPUDirectRDMAWritesOrdering for the numerical values returned here. */ + cudaDevAttrMemoryPoolSupportedHandleTypes = 119, /**< Handle types supported with mempool based IPC */ + cudaDevAttrClusterLaunch = 120, /**< Indicates device supports cluster launch */ + cudaDevAttrDeferredMappingCudaArraySupported = 121, /**< Device supports deferred mapping CUDA arrays and CUDA mipmapped arrays */ + cudaDevAttrReserved122 = 122, + cudaDevAttrReserved123 = 123, + cudaDevAttrReserved124 = 124, + cudaDevAttrIpcEventSupport = 125, /**< Device supports IPC Events. */ + cudaDevAttrMemSyncDomainCount = 126, /**< Number of memory synchronization domains the device supports. */ + cudaDevAttrReserved127 = 127, + cudaDevAttrReserved128 = 128, + cudaDevAttrReserved129 = 129, + cudaDevAttrNumaConfig = 130, /**< NUMA configuration of a device: value is of type cudaDeviceNumaConfig enum */ + cudaDevAttrNumaId = 131, /**< NUMA node ID of the GPU memory */ + cudaDevAttrReserved132 = 132, + cudaDevAttrMpsEnabled = 133, /**< Contexts created on this device will be shared via MPS */ + cudaDevAttrHostNumaId = 134, /**< NUMA ID of the host node closest to the device. Returns -1 when system does not support NUMA. */ + cudaDevAttrMax +}; + +/** + * CUDA memory pool attributes + */ +enum __device_builtin__ cudaMemPoolAttr +{ + /** + * (value type = int) + * Allow cuMemAllocAsync to use memory asynchronously freed + * in another streams as long as a stream ordering dependency + * of the allocating stream on the free action exists. + * Cuda events and null stream interactions can create the required + * stream ordered dependencies. (default enabled) + */ + cudaMemPoolReuseFollowEventDependencies = 0x1, + + /** + * (value type = int) + * Allow reuse of already completed frees when there is no dependency + * between the free and allocation. (default enabled) + */ + cudaMemPoolReuseAllowOpportunistic = 0x2, + + /** + * (value type = int) + * Allow cuMemAllocAsync to insert new stream dependencies + * in order to establish the stream ordering required to reuse + * a piece of memory released by cuFreeAsync (default enabled). + */ + cudaMemPoolReuseAllowInternalDependencies = 0x3, + + + /** + * (value type = cuuint64_t) + * Amount of reserved memory in bytes to hold onto before trying + * to release memory back to the OS. When more than the release + * threshold bytes of memory are held by the memory pool, the + * allocator will try to release memory back to the OS on the + * next call to stream, event or context synchronize. (default 0) + */ + cudaMemPoolAttrReleaseThreshold = 0x4, + + /** + * (value type = cuuint64_t) + * Amount of backing memory currently allocated for the mempool. + */ + cudaMemPoolAttrReservedMemCurrent = 0x5, + + /** + * (value type = cuuint64_t) + * High watermark of backing memory allocated for the mempool since the + * last time it was reset. High watermark can only be reset to zero. + */ + cudaMemPoolAttrReservedMemHigh = 0x6, + + /** + * (value type = cuuint64_t) + * Amount of memory from the pool that is currently in use by the application. + */ + cudaMemPoolAttrUsedMemCurrent = 0x7, + + /** + * (value type = cuuint64_t) + * High watermark of the amount of memory from the pool that was in use by the application since + * the last time it was reset. High watermark can only be reset to zero. + */ + cudaMemPoolAttrUsedMemHigh = 0x8 +}; + +/** + * Specifies the type of location + */ +enum __device_builtin__ cudaMemLocationType { + cudaMemLocationTypeInvalid = 0, + cudaMemLocationTypeDevice = 1 /**< Location is a device location, thus id is a device ordinal */ + , cudaMemLocationTypeHost = 2 /**< Location is host, id is ignored */ + , cudaMemLocationTypeHostNuma = 3 /**< Location is a host NUMA node, thus id is a host NUMA node id */ + , cudaMemLocationTypeHostNumaCurrent = 4 /**< Location is the host NUMA node closest to the current thread's CPU, id is ignored */ +}; + +/** + * Specifies a memory location. + * + * To specify a gpu, set type = ::cudaMemLocationTypeDevice and set id = the gpu's device ordinal. + * To specify a cpu NUMA node, set type = ::cudaMemLocationTypeHostNuma and set id = host NUMA node id. + */ +struct __device_builtin__ cudaMemLocation { + enum cudaMemLocationType type; /**< Specifies the location type, which modifies the meaning of id. */ + int id; /**< identifier for a given this location's ::CUmemLocationType. */ +}; + +/** + * Specifies the memory protection flags for mapping. + */ +enum __device_builtin__ cudaMemAccessFlags { + cudaMemAccessFlagsProtNone = 0, /**< Default, make the address range not accessible */ + cudaMemAccessFlagsProtRead = 1, /**< Make the address range read accessible */ + cudaMemAccessFlagsProtReadWrite = 3 /**< Make the address range read-write accessible */ +}; + +/** + * Memory access descriptor + */ +struct __device_builtin__ cudaMemAccessDesc { + struct cudaMemLocation location; /**< Location on which the request is to change it's accessibility */ + enum cudaMemAccessFlags flags; /**< ::CUmemProt accessibility flags to set on the request */ +}; + +/** + * Defines the allocation types available + */ +enum __device_builtin__ cudaMemAllocationType { + cudaMemAllocationTypeInvalid = 0x0, + /** This allocation type is 'pinned', i.e. cannot migrate from its current + * location while the application is actively using it + */ + cudaMemAllocationTypePinned = 0x1, + cudaMemAllocationTypeMax = 0x7FFFFFFF +}; + +/** + * Flags for specifying particular handle types + */ +enum __device_builtin__ cudaMemAllocationHandleType { + cudaMemHandleTypeNone = 0x0, /**< Does not allow any export mechanism. > */ + cudaMemHandleTypePosixFileDescriptor = 0x1, /**< Allows a file descriptor to be used for exporting. Permitted only on POSIX systems. (int) */ + cudaMemHandleTypeWin32 = 0x2, /**< Allows a Win32 NT handle to be used for exporting. (HANDLE) */ + cudaMemHandleTypeWin32Kmt = 0x4, /**< Allows a Win32 KMT handle to be used for exporting. (D3DKMT_HANDLE) */ + cudaMemHandleTypeFabric = 0x8 /**< Allows a fabric handle to be used for exporting. (cudaMemFabricHandle_t) */ +}; + +/** + * Specifies the properties of allocations made from the pool. + */ +struct __device_builtin__ cudaMemPoolProps { + enum cudaMemAllocationType allocType; /**< Allocation type. Currently must be specified as cudaMemAllocationTypePinned */ + enum cudaMemAllocationHandleType handleTypes; /**< Handle types that will be supported by allocations from the pool. */ + struct cudaMemLocation location; /**< Location allocations should reside. */ + /** + * Windows-specific LPSECURITYATTRIBUTES required when + * ::cudaMemHandleTypeWin32 is specified. This security attribute defines + * the scope of which exported allocations may be tranferred to other + * processes. In all other cases, this field is required to be zero. + */ + void *win32SecurityAttributes; + size_t maxSize; /**< Maximum pool size. When set to 0, defaults to a system dependent value.*/ + unsigned char reserved[56]; /**< reserved for future use, must be 0 */ +}; + +/** + * Opaque data for exporting a pool allocation + */ +struct __device_builtin__ cudaMemPoolPtrExportData { + unsigned char reserved[64]; +}; + +/** + * Memory allocation node parameters + */ +struct __device_builtin__ cudaMemAllocNodeParams { + /** + * in: location where the allocation should reside (specified in ::location). + * ::handleTypes must be ::cudaMemHandleTypeNone. IPC is not supported. + */ + struct cudaMemPoolProps poolProps; /**< in: array of memory access descriptors. Used to describe peer GPU access */ + const struct cudaMemAccessDesc *accessDescs; /**< in: number of memory access descriptors. Must not exceed the number of GPUs. */ + size_t accessDescCount; /**< in: Number of `accessDescs`s */ + size_t bytesize; /**< in: size in bytes of the requested allocation */ + void *dptr; /**< out: address of the allocation returned by CUDA */ +}; + +/** + * Memory allocation node parameters + */ +struct __device_builtin__ cudaMemAllocNodeParamsV2 { + /** + * in: location where the allocation should reside (specified in ::location). + * ::handleTypes must be ::cudaMemHandleTypeNone. IPC is not supported. + */ + struct cudaMemPoolProps poolProps; /**< in: array of memory access descriptors. Used to describe peer GPU access */ + const struct cudaMemAccessDesc *accessDescs; /**< in: number of memory access descriptors. Must not exceed the number of GPUs. */ + size_t accessDescCount; /**< in: Number of `accessDescs`s */ + size_t bytesize; /**< in: size in bytes of the requested allocation */ + void *dptr; /**< out: address of the allocation returned by CUDA */ +}; + +/** + * Memory free node parameters + */ +struct __device_builtin__ cudaMemFreeNodeParams { + void *dptr; /**< in: the pointer to free */ +}; + +/** + * Graph memory attributes + */ +enum __device_builtin__ cudaGraphMemAttributeType { + /** + * (value type = cuuint64_t) + * Amount of memory, in bytes, currently associated with graphs. + */ + cudaGraphMemAttrUsedMemCurrent = 0x0, + + /** + * (value type = cuuint64_t) + * High watermark of memory, in bytes, associated with graphs since the + * last time it was reset. High watermark can only be reset to zero. + */ + cudaGraphMemAttrUsedMemHigh = 0x1, + + /** + * (value type = cuuint64_t) + * Amount of memory, in bytes, currently allocated for use by + * the CUDA graphs asynchronous allocator. + */ + cudaGraphMemAttrReservedMemCurrent = 0x2, + + /** + * (value type = cuuint64_t) + * High watermark of memory, in bytes, currently allocated for use by + * the CUDA graphs asynchronous allocator. + */ + cudaGraphMemAttrReservedMemHigh = 0x3 +}; + +/** + * CUDA device P2P attributes + */ + +enum __device_builtin__ cudaDeviceP2PAttr { + cudaDevP2PAttrPerformanceRank = 1, /**< A relative value indicating the performance of the link between two devices */ + cudaDevP2PAttrAccessSupported = 2, /**< Peer access is enabled */ + cudaDevP2PAttrNativeAtomicSupported = 3, /**< Native atomic operation over the link supported */ + cudaDevP2PAttrCudaArrayAccessSupported = 4 /**< Accessing CUDA arrays over the link supported */ +}; + +/** + * CUDA UUID types + */ +#ifndef CU_UUID_HAS_BEEN_DEFINED +#define CU_UUID_HAS_BEEN_DEFINED +struct __device_builtin__ CUuuid_st { /**< CUDA definition of UUID */ + char bytes[16]; +}; +typedef __device_builtin__ struct CUuuid_st CUuuid; +#endif +typedef __device_builtin__ struct CUuuid_st cudaUUID_t; + +/** + * CUDA device properties + */ +struct __device_builtin__ cudaDeviceProp +{ + char name[256]; /**< ASCII string identifying device */ + cudaUUID_t uuid; /**< 16-byte unique identifier */ + char luid[8]; /**< 8-byte locally unique identifier. Value is undefined on TCC and non-Windows platforms */ + unsigned int luidDeviceNodeMask; /**< LUID device node mask. Value is undefined on TCC and non-Windows platforms */ + size_t totalGlobalMem; /**< Global memory available on device in bytes */ + size_t sharedMemPerBlock; /**< Shared memory available per block in bytes */ + int regsPerBlock; /**< 32-bit registers available per block */ + int warpSize; /**< Warp size in threads */ + size_t memPitch; /**< Maximum pitch in bytes allowed by memory copies */ + int maxThreadsPerBlock; /**< Maximum number of threads per block */ + int maxThreadsDim[3]; /**< Maximum size of each dimension of a block */ + int maxGridSize[3]; /**< Maximum size of each dimension of a grid */ + int clockRate; /**< Deprecated, Clock frequency in kilohertz */ + size_t totalConstMem; /**< Constant memory available on device in bytes */ + int major; /**< Major compute capability */ + int minor; /**< Minor compute capability */ + size_t textureAlignment; /**< Alignment requirement for textures */ + size_t texturePitchAlignment; /**< Pitch alignment requirement for texture references bound to pitched memory */ + int deviceOverlap; /**< Device can concurrently copy memory and execute a kernel. Deprecated. Use instead asyncEngineCount. */ + int multiProcessorCount; /**< Number of multiprocessors on device */ + int kernelExecTimeoutEnabled; /**< Deprecated, Specified whether there is a run time limit on kernels */ + int integrated; /**< Device is integrated as opposed to discrete */ + int canMapHostMemory; /**< Device can map host memory with cudaHostAlloc/cudaHostGetDevicePointer */ + int computeMode; /**< Deprecated, Compute mode (See ::cudaComputeMode) */ + int maxTexture1D; /**< Maximum 1D texture size */ + int maxTexture1DMipmap; /**< Maximum 1D mipmapped texture size */ + int maxTexture1DLinear; /**< Deprecated, do not use. Use cudaDeviceGetTexture1DLinearMaxWidth() or cuDeviceGetTexture1DLinearMaxWidth() instead. */ + int maxTexture2D[2]; /**< Maximum 2D texture dimensions */ + int maxTexture2DMipmap[2]; /**< Maximum 2D mipmapped texture dimensions */ + int maxTexture2DLinear[3]; /**< Maximum dimensions (width, height, pitch) for 2D textures bound to pitched memory */ + int maxTexture2DGather[2]; /**< Maximum 2D texture dimensions if texture gather operations have to be performed */ + int maxTexture3D[3]; /**< Maximum 3D texture dimensions */ + int maxTexture3DAlt[3]; /**< Maximum alternate 3D texture dimensions */ + int maxTextureCubemap; /**< Maximum Cubemap texture dimensions */ + int maxTexture1DLayered[2]; /**< Maximum 1D layered texture dimensions */ + int maxTexture2DLayered[3]; /**< Maximum 2D layered texture dimensions */ + int maxTextureCubemapLayered[2];/**< Maximum Cubemap layered texture dimensions */ + int maxSurface1D; /**< Maximum 1D surface size */ + int maxSurface2D[2]; /**< Maximum 2D surface dimensions */ + int maxSurface3D[3]; /**< Maximum 3D surface dimensions */ + int maxSurface1DLayered[2]; /**< Maximum 1D layered surface dimensions */ + int maxSurface2DLayered[3]; /**< Maximum 2D layered surface dimensions */ + int maxSurfaceCubemap; /**< Maximum Cubemap surface dimensions */ + int maxSurfaceCubemapLayered[2];/**< Maximum Cubemap layered surface dimensions */ + size_t surfaceAlignment; /**< Alignment requirements for surfaces */ + int concurrentKernels; /**< Device can possibly execute multiple kernels concurrently */ + int ECCEnabled; /**< Device has ECC support enabled */ + int pciBusID; /**< PCI bus ID of the device */ + int pciDeviceID; /**< PCI device ID of the device */ + int pciDomainID; /**< PCI domain ID of the device */ + int tccDriver; /**< 1 if device is a Tesla device using TCC driver, 0 otherwise */ + int asyncEngineCount; /**< Number of asynchronous engines */ + int unifiedAddressing; /**< Device shares a unified address space with the host */ + int memoryClockRate; /**< Deprecated, Peak memory clock frequency in kilohertz */ + int memoryBusWidth; /**< Global memory bus width in bits */ + int l2CacheSize; /**< Size of L2 cache in bytes */ + int persistingL2CacheMaxSize; /**< Device's maximum l2 persisting lines capacity setting in bytes */ + int maxThreadsPerMultiProcessor;/**< Maximum resident threads per multiprocessor */ + int streamPrioritiesSupported; /**< Device supports stream priorities */ + int globalL1CacheSupported; /**< Device supports caching globals in L1 */ + int localL1CacheSupported; /**< Device supports caching locals in L1 */ + size_t sharedMemPerMultiprocessor; /**< Shared memory available per multiprocessor in bytes */ + int regsPerMultiprocessor; /**< 32-bit registers available per multiprocessor */ + int managedMemory; /**< Device supports allocating managed memory on this system */ + int isMultiGpuBoard; /**< Device is on a multi-GPU board */ + int multiGpuBoardGroupID; /**< Unique identifier for a group of devices on the same multi-GPU board */ + int hostNativeAtomicSupported; /**< Link between the device and the host supports native atomic operations */ + int singleToDoublePrecisionPerfRatio; /**< Deprecated, Ratio of single precision performance (in floating-point operations per second) to double precision performance */ + int pageableMemoryAccess; /**< Device supports coherently accessing pageable memory without calling cudaHostRegister on it */ + int concurrentManagedAccess; /**< Device can coherently access managed memory concurrently with the CPU */ + int computePreemptionSupported; /**< Device supports Compute Preemption */ + int canUseHostPointerForRegisteredMem; /**< Device can access host registered memory at the same virtual address as the CPU */ + int cooperativeLaunch; /**< Device supports launching cooperative kernels via ::cudaLaunchCooperativeKernel */ + int cooperativeMultiDeviceLaunch; /**< Deprecated, cudaLaunchCooperativeKernelMultiDevice is deprecated. */ + size_t sharedMemPerBlockOptin; /**< Per device maximum shared memory per block usable by special opt in */ + int pageableMemoryAccessUsesHostPageTables; /**< Device accesses pageable memory via the host's page tables */ + int directManagedMemAccessFromHost; /**< Host can directly access managed memory on the device without migration. */ + int maxBlocksPerMultiProcessor; /**< Maximum number of resident blocks per multiprocessor */ + int accessPolicyMaxWindowSize; /**< The maximum value of ::cudaAccessPolicyWindow::num_bytes. */ + size_t reservedSharedMemPerBlock; /**< Shared memory reserved by CUDA driver per block in bytes */ + int hostRegisterSupported; /**< Device supports host memory registration via ::cudaHostRegister. */ + int sparseCudaArraySupported; /**< 1 if the device supports sparse CUDA arrays and sparse CUDA mipmapped arrays, 0 otherwise */ + int hostRegisterReadOnlySupported; /**< Device supports using the ::cudaHostRegister flag cudaHostRegisterReadOnly to register memory that must be mapped as read-only to the GPU */ + int timelineSemaphoreInteropSupported; /**< External timeline semaphore interop is supported on the device */ + int memoryPoolsSupported; /**< 1 if the device supports using the cudaMallocAsync and cudaMemPool family of APIs, 0 otherwise */ + int gpuDirectRDMASupported; /**< 1 if the device supports GPUDirect RDMA APIs, 0 otherwise */ + unsigned int gpuDirectRDMAFlushWritesOptions; /**< Bitmask to be interpreted according to the ::cudaFlushGPUDirectRDMAWritesOptions enum */ + int gpuDirectRDMAWritesOrdering;/**< See the ::cudaGPUDirectRDMAWritesOrdering enum for numerical values */ + unsigned int memoryPoolSupportedHandleTypes; /**< Bitmask of handle types supported with mempool-based IPC */ + int deferredMappingCudaArraySupported; /**< 1 if the device supports deferred mapping CUDA arrays and CUDA mipmapped arrays */ + int ipcEventSupported; /**< Device supports IPC Events. */ + int clusterLaunch; /**< Indicates device supports cluster launch */ + int unifiedFunctionPointers; /**< Indicates device supports unified pointers */ + int reserved2[2]; + int reserved1[1]; /**< Reserved for future use */ + int reserved[60]; /**< Reserved for future use */ +}; + +/** + * CUDA IPC Handle Size + */ +#define CUDA_IPC_HANDLE_SIZE 64 + +/** + * CUDA IPC event handle + */ +typedef __device_builtin__ struct __device_builtin__ cudaIpcEventHandle_st +{ + char reserved[CUDA_IPC_HANDLE_SIZE]; +}cudaIpcEventHandle_t; + +/** + * CUDA IPC memory handle + */ +typedef __device_builtin__ struct __device_builtin__ cudaIpcMemHandle_st +{ + char reserved[CUDA_IPC_HANDLE_SIZE]; +}cudaIpcMemHandle_t; + +/* + * CUDA Mem Fabric Handle + */ +typedef __device_builtin__ struct __device_builtin__ cudaMemFabricHandle_st +{ + char reserved[CUDA_IPC_HANDLE_SIZE]; +}cudaMemFabricHandle_t; + +/** + * External memory handle types + */ +enum __device_builtin__ cudaExternalMemoryHandleType { + /** + * Handle is an opaque file descriptor + */ + cudaExternalMemoryHandleTypeOpaqueFd = 1, + /** + * Handle is an opaque shared NT handle + */ + cudaExternalMemoryHandleTypeOpaqueWin32 = 2, + /** + * Handle is an opaque, globally shared handle + */ + cudaExternalMemoryHandleTypeOpaqueWin32Kmt = 3, + /** + * Handle is a D3D12 heap object + */ + cudaExternalMemoryHandleTypeD3D12Heap = 4, + /** + * Handle is a D3D12 committed resource + */ + cudaExternalMemoryHandleTypeD3D12Resource = 5, + /** + * Handle is a shared NT handle to a D3D11 resource + */ + cudaExternalMemoryHandleTypeD3D11Resource = 6, + /** + * Handle is a globally shared handle to a D3D11 resource + */ + cudaExternalMemoryHandleTypeD3D11ResourceKmt = 7, + /** + * Handle is an NvSciBuf object + */ + cudaExternalMemoryHandleTypeNvSciBuf = 8 +}; + +/** + * Indicates that the external memory object is a dedicated resource + */ +#define cudaExternalMemoryDedicated 0x1 + +/** When the /p flags parameter of ::cudaExternalSemaphoreSignalParams + * contains this flag, it indicates that signaling an external semaphore object + * should skip performing appropriate memory synchronization operations over all + * the external memory objects that are imported as ::cudaExternalMemoryHandleTypeNvSciBuf, + * which otherwise are performed by default to ensure data coherency with other + * importers of the same NvSciBuf memory objects. + */ +#define cudaExternalSemaphoreSignalSkipNvSciBufMemSync 0x01 + +/** When the /p flags parameter of ::cudaExternalSemaphoreWaitParams + * contains this flag, it indicates that waiting an external semaphore object + * should skip performing appropriate memory synchronization operations over all + * the external memory objects that are imported as ::cudaExternalMemoryHandleTypeNvSciBuf, + * which otherwise are performed by default to ensure data coherency with other + * importers of the same NvSciBuf memory objects. + */ +#define cudaExternalSemaphoreWaitSkipNvSciBufMemSync 0x02 + +/** + * When /p flags of ::cudaDeviceGetNvSciSyncAttributes is set to this, + * it indicates that application need signaler specific NvSciSyncAttr + * to be filled by ::cudaDeviceGetNvSciSyncAttributes. + */ +#define cudaNvSciSyncAttrSignal 0x1 + +/** + * When /p flags of ::cudaDeviceGetNvSciSyncAttributes is set to this, + * it indicates that application need waiter specific NvSciSyncAttr + * to be filled by ::cudaDeviceGetNvSciSyncAttributes. + */ +#define cudaNvSciSyncAttrWait 0x2 + +/** + * External memory handle descriptor + */ +struct __device_builtin__ cudaExternalMemoryHandleDesc { + /** + * Type of the handle + */ + enum cudaExternalMemoryHandleType type; + union { + /** + * File descriptor referencing the memory object. Valid + * when type is + * ::cudaExternalMemoryHandleTypeOpaqueFd + */ + int fd; + /** + * Win32 handle referencing the semaphore object. Valid when + * type is one of the following: + * - ::cudaExternalMemoryHandleTypeOpaqueWin32 + * - ::cudaExternalMemoryHandleTypeOpaqueWin32Kmt + * - ::cudaExternalMemoryHandleTypeD3D12Heap + * - ::cudaExternalMemoryHandleTypeD3D12Resource + * - ::cudaExternalMemoryHandleTypeD3D11Resource + * - ::cudaExternalMemoryHandleTypeD3D11ResourceKmt + * Exactly one of 'handle' and 'name' must be non-NULL. If + * type is one of the following: + * ::cudaExternalMemoryHandleTypeOpaqueWin32Kmt + * ::cudaExternalMemoryHandleTypeD3D11ResourceKmt + * then 'name' must be NULL. + */ + struct { + /** + * Valid NT handle. Must be NULL if 'name' is non-NULL + */ + void *handle; + /** + * Name of a valid memory object. + * Must be NULL if 'handle' is non-NULL. + */ + const void *name; + } win32; + /** + * A handle representing NvSciBuf Object. Valid when type + * is ::cudaExternalMemoryHandleTypeNvSciBuf + */ + const void *nvSciBufObject; + } handle; + /** + * Size of the memory allocation + */ + unsigned long long size; + /** + * Flags must either be zero or ::cudaExternalMemoryDedicated + */ + unsigned int flags; +}; + +/** + * External memory buffer descriptor + */ +struct __device_builtin__ cudaExternalMemoryBufferDesc { + /** + * Offset into the memory object where the buffer's base is + */ + unsigned long long offset; + /** + * Size of the buffer + */ + unsigned long long size; + /** + * Flags reserved for future use. Must be zero. + */ + unsigned int flags; +}; + +/** + * External memory mipmap descriptor + */ +struct __device_builtin__ cudaExternalMemoryMipmappedArrayDesc { + /** + * Offset into the memory object where the base level of the + * mipmap chain is. + */ + unsigned long long offset; + /** + * Format of base level of the mipmap chain + */ + struct cudaChannelFormatDesc formatDesc; + /** + * Dimensions of base level of the mipmap chain + */ + struct cudaExtent extent; + /** + * Flags associated with CUDA mipmapped arrays. + * See ::cudaMallocMipmappedArray + */ + unsigned int flags; + /** + * Total number of levels in the mipmap chain + */ + unsigned int numLevels; +}; + +/** + * External semaphore handle types + */ +enum __device_builtin__ cudaExternalSemaphoreHandleType { + /** + * Handle is an opaque file descriptor + */ + cudaExternalSemaphoreHandleTypeOpaqueFd = 1, + /** + * Handle is an opaque shared NT handle + */ + cudaExternalSemaphoreHandleTypeOpaqueWin32 = 2, + /** + * Handle is an opaque, globally shared handle + */ + cudaExternalSemaphoreHandleTypeOpaqueWin32Kmt = 3, + /** + * Handle is a shared NT handle referencing a D3D12 fence object + */ + cudaExternalSemaphoreHandleTypeD3D12Fence = 4, + /** + * Handle is a shared NT handle referencing a D3D11 fence object + */ + cudaExternalSemaphoreHandleTypeD3D11Fence = 5, + /** + * Opaque handle to NvSciSync Object + */ + cudaExternalSemaphoreHandleTypeNvSciSync = 6, + /** + * Handle is a shared NT handle referencing a D3D11 keyed mutex object + */ + cudaExternalSemaphoreHandleTypeKeyedMutex = 7, + /** + * Handle is a shared KMT handle referencing a D3D11 keyed mutex object + */ + cudaExternalSemaphoreHandleTypeKeyedMutexKmt = 8, + /** + * Handle is an opaque handle file descriptor referencing a timeline semaphore + */ + cudaExternalSemaphoreHandleTypeTimelineSemaphoreFd = 9, + /** + * Handle is an opaque handle file descriptor referencing a timeline semaphore + */ + cudaExternalSemaphoreHandleTypeTimelineSemaphoreWin32 = 10 +}; + +/** + * External semaphore handle descriptor + */ +struct __device_builtin__ cudaExternalSemaphoreHandleDesc { + /** + * Type of the handle + */ + enum cudaExternalSemaphoreHandleType type; + union { + /** + * File descriptor referencing the semaphore object. Valid when + * type is one of the following: + * - ::cudaExternalSemaphoreHandleTypeOpaqueFd + * - ::cudaExternalSemaphoreHandleTypeTimelineSemaphoreFd + */ + int fd; + /** + * Win32 handle referencing the semaphore object. Valid when + * type is one of the following: + * - ::cudaExternalSemaphoreHandleTypeOpaqueWin32 + * - ::cudaExternalSemaphoreHandleTypeOpaqueWin32Kmt + * - ::cudaExternalSemaphoreHandleTypeD3D12Fence + * - ::cudaExternalSemaphoreHandleTypeD3D11Fence + * - ::cudaExternalSemaphoreHandleTypeKeyedMutex + * - ::cudaExternalSemaphoreHandleTypeTimelineSemaphoreWin32 + * Exactly one of 'handle' and 'name' must be non-NULL. If + * type is one of the following: + * ::cudaExternalSemaphoreHandleTypeOpaqueWin32Kmt + * ::cudaExternalSemaphoreHandleTypeKeyedMutexKmt + * then 'name' must be NULL. + */ + struct { + /** + * Valid NT handle. Must be NULL if 'name' is non-NULL + */ + void *handle; + /** + * Name of a valid synchronization primitive. + * Must be NULL if 'handle' is non-NULL. + */ + const void *name; + } win32; + /** + * Valid NvSciSyncObj. Must be non NULL + */ + const void* nvSciSyncObj; + } handle; + /** + * Flags reserved for the future. Must be zero. + */ + unsigned int flags; +}; + +/** + * External semaphore signal parameters(deprecated) + */ +struct __device_builtin__ cudaExternalSemaphoreSignalParams_v1 { + struct { + /** + * Parameters for fence objects + */ + struct { + /** + * Value of fence to be signaled + */ + unsigned long long value; + } fence; + union { + /** + * Pointer to NvSciSyncFence. Valid if ::cudaExternalSemaphoreHandleType + * is of type ::cudaExternalSemaphoreHandleTypeNvSciSync. + */ + void *fence; + unsigned long long reserved; + } nvSciSync; + /** + * Parameters for keyed mutex objects + */ + struct { + /* + * Value of key to release the mutex with + */ + unsigned long long key; + } keyedMutex; + } params; + /** + * Only when ::cudaExternalSemaphoreSignalParams is used to + * signal a ::cudaExternalSemaphore_t of type + * ::cudaExternalSemaphoreHandleTypeNvSciSync, the valid flag is + * ::cudaExternalSemaphoreSignalSkipNvSciBufMemSync: which indicates + * that while signaling the ::cudaExternalSemaphore_t, no memory + * synchronization operations should be performed for any external memory + * object imported as ::cudaExternalMemoryHandleTypeNvSciBuf. + * For all other types of ::cudaExternalSemaphore_t, flags must be zero. + */ + unsigned int flags; +}; + +/** +* External semaphore wait parameters(deprecated) +*/ +struct __device_builtin__ cudaExternalSemaphoreWaitParams_v1 { + struct { + /** + * Parameters for fence objects + */ + struct { + /** + * Value of fence to be waited on + */ + unsigned long long value; + } fence; + union { + /** + * Pointer to NvSciSyncFence. Valid if ::cudaExternalSemaphoreHandleType + * is of type ::cudaExternalSemaphoreHandleTypeNvSciSync. + */ + void *fence; + unsigned long long reserved; + } nvSciSync; + /** + * Parameters for keyed mutex objects + */ + struct { + /** + * Value of key to acquire the mutex with + */ + unsigned long long key; + /** + * Timeout in milliseconds to wait to acquire the mutex + */ + unsigned int timeoutMs; + } keyedMutex; + } params; + /** + * Only when ::cudaExternalSemaphoreSignalParams is used to + * signal a ::cudaExternalSemaphore_t of type + * ::cudaExternalSemaphoreHandleTypeNvSciSync, the valid flag is + * ::cudaExternalSemaphoreSignalSkipNvSciBufMemSync: which indicates + * that while waiting for the ::cudaExternalSemaphore_t, no memory + * synchronization operations should be performed for any external memory + * object imported as ::cudaExternalMemoryHandleTypeNvSciBuf. + * For all other types of ::cudaExternalSemaphore_t, flags must be zero. + */ + unsigned int flags; +}; + +/** + * External semaphore signal parameters, compatible with driver type + */ +struct __device_builtin__ cudaExternalSemaphoreSignalParams{ + struct { + /** + * Parameters for fence objects + */ + struct { + /** + * Value of fence to be signaled + */ + unsigned long long value; + } fence; + union { + /** + * Pointer to NvSciSyncFence. Valid if ::cudaExternalSemaphoreHandleType + * is of type ::cudaExternalSemaphoreHandleTypeNvSciSync. + */ + void *fence; + unsigned long long reserved; + } nvSciSync; + /** + * Parameters for keyed mutex objects + */ + struct { + /* + * Value of key to release the mutex with + */ + unsigned long long key; + } keyedMutex; + unsigned int reserved[12]; + } params; + /** + * Only when ::cudaExternalSemaphoreSignalParams is used to + * signal a ::cudaExternalSemaphore_t of type + * ::cudaExternalSemaphoreHandleTypeNvSciSync, the valid flag is + * ::cudaExternalSemaphoreSignalSkipNvSciBufMemSync: which indicates + * that while signaling the ::cudaExternalSemaphore_t, no memory + * synchronization operations should be performed for any external memory + * object imported as ::cudaExternalMemoryHandleTypeNvSciBuf. + * For all other types of ::cudaExternalSemaphore_t, flags must be zero. + */ + unsigned int flags; + unsigned int reserved[16]; +}; + +/** + * External semaphore wait parameters, compatible with driver type + */ +struct __device_builtin__ cudaExternalSemaphoreWaitParams { + struct { + /** + * Parameters for fence objects + */ + struct { + /** + * Value of fence to be waited on + */ + unsigned long long value; + } fence; + union { + /** + * Pointer to NvSciSyncFence. Valid if ::cudaExternalSemaphoreHandleType + * is of type ::cudaExternalSemaphoreHandleTypeNvSciSync. + */ + void *fence; + unsigned long long reserved; + } nvSciSync; + /** + * Parameters for keyed mutex objects + */ + struct { + /** + * Value of key to acquire the mutex with + */ + unsigned long long key; + /** + * Timeout in milliseconds to wait to acquire the mutex + */ + unsigned int timeoutMs; + } keyedMutex; + unsigned int reserved[10]; + } params; + /** + * Only when ::cudaExternalSemaphoreSignalParams is used to + * signal a ::cudaExternalSemaphore_t of type + * ::cudaExternalSemaphoreHandleTypeNvSciSync, the valid flag is + * ::cudaExternalSemaphoreSignalSkipNvSciBufMemSync: which indicates + * that while waiting for the ::cudaExternalSemaphore_t, no memory + * synchronization operations should be performed for any external memory + * object imported as ::cudaExternalMemoryHandleTypeNvSciBuf. + * For all other types of ::cudaExternalSemaphore_t, flags must be zero. + */ + unsigned int flags; + unsigned int reserved[16]; +}; + +/******************************************************************************* +* * +* SHORTHAND TYPE DEFINITION USED BY RUNTIME API * +* * +*******************************************************************************/ + +/** + * CUDA Error types + */ +typedef __device_builtin__ enum cudaError cudaError_t; + +/** + * CUDA stream + */ +typedef __device_builtin__ struct CUstream_st *cudaStream_t; + +/** + * CUDA event types + */ +typedef __device_builtin__ struct CUevent_st *cudaEvent_t; + +/** + * CUDA graphics resource types + */ +typedef __device_builtin__ struct cudaGraphicsResource *cudaGraphicsResource_t; + +/** + * CUDA external memory + */ +typedef __device_builtin__ struct CUexternalMemory_st *cudaExternalMemory_t; + +/** + * CUDA external semaphore + */ +typedef __device_builtin__ struct CUexternalSemaphore_st *cudaExternalSemaphore_t; + +/** + * CUDA graph + */ +typedef __device_builtin__ struct CUgraph_st *cudaGraph_t; + +/** + * CUDA graph node. + */ +typedef __device_builtin__ struct CUgraphNode_st *cudaGraphNode_t; + +/** + * CUDA user object for graphs + */ +typedef __device_builtin__ struct CUuserObject_st *cudaUserObject_t; + +/** + * CUDA handle for conditional graph nodes + */ +typedef __device_builtin__ unsigned long long cudaGraphConditionalHandle; + +/** + * CUDA function + */ +typedef __device_builtin__ struct CUfunc_st *cudaFunction_t; + +/** + * CUDA kernel + */ +typedef __device_builtin__ struct CUkern_st *cudaKernel_t; + +/** + * CUDA memory pool + */ +typedef __device_builtin__ struct CUmemPoolHandle_st *cudaMemPool_t; + +/** + * CUDA cooperative group scope + */ +enum __device_builtin__ cudaCGScope { + cudaCGScopeInvalid = 0, /**< Invalid cooperative group scope */ + cudaCGScopeGrid = 1, /**< Scope represented by a grid_group */ + cudaCGScopeMultiGrid = 2 /**< Scope represented by a multi_grid_group */ +}; + +/** + * CUDA launch parameters + */ +struct __device_builtin__ cudaLaunchParams +{ + void *func; /**< Device function symbol */ + dim3 gridDim; /**< Grid dimentions */ + dim3 blockDim; /**< Block dimentions */ + void **args; /**< Arguments */ + size_t sharedMem; /**< Shared memory */ + cudaStream_t stream; /**< Stream identifier */ +}; + +/** + * CUDA GPU kernel node parameters + */ +struct __device_builtin__ cudaKernelNodeParams { + void* func; /**< Kernel to launch */ + dim3 gridDim; /**< Grid dimensions */ + dim3 blockDim; /**< Block dimensions */ + unsigned int sharedMemBytes; /**< Dynamic shared-memory size per thread block in bytes */ + void **kernelParams; /**< Array of pointers to individual kernel arguments*/ + void **extra; /**< Pointer to kernel arguments in the "extra" format */ +}; + +/** + * CUDA GPU kernel node parameters + */ +struct __device_builtin__ cudaKernelNodeParamsV2 { + void* func; /**< Kernel to launch */ + #if !defined(__cplusplus) || __cplusplus >= 201103L + dim3 gridDim; /**< Grid dimensions */ + dim3 blockDim; /**< Block dimensions */ + #else + /* Union members cannot have nontrivial constructors until C++11. */ + uint3 gridDim; /**< Grid dimensions */ + uint3 blockDim; /**< Block dimensions */ + #endif + unsigned int sharedMemBytes; /**< Dynamic shared-memory size per thread block in bytes */ + void **kernelParams; /**< Array of pointers to individual kernel arguments*/ + void **extra; /**< Pointer to kernel arguments in the "extra" format */ +}; + +/** + * External semaphore signal node parameters + */ +struct __device_builtin__ cudaExternalSemaphoreSignalNodeParams { + cudaExternalSemaphore_t* extSemArray; /**< Array of external semaphore handles. */ + const struct cudaExternalSemaphoreSignalParams* paramsArray; /**< Array of external semaphore signal parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +}; + +/** + * External semaphore signal node parameters + */ +struct __device_builtin__ cudaExternalSemaphoreSignalNodeParamsV2 { + cudaExternalSemaphore_t* extSemArray; /**< Array of external semaphore handles. */ + const struct cudaExternalSemaphoreSignalParams* paramsArray; /**< Array of external semaphore signal parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +}; + +/** + * External semaphore wait node parameters + */ +struct __device_builtin__ cudaExternalSemaphoreWaitNodeParams { + cudaExternalSemaphore_t* extSemArray; /**< Array of external semaphore handles. */ + const struct cudaExternalSemaphoreWaitParams* paramsArray; /**< Array of external semaphore wait parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +}; + +/** + * External semaphore wait node parameters + */ +struct __device_builtin__ cudaExternalSemaphoreWaitNodeParamsV2 { + cudaExternalSemaphore_t* extSemArray; /**< Array of external semaphore handles. */ + const struct cudaExternalSemaphoreWaitParams* paramsArray; /**< Array of external semaphore wait parameters. */ + unsigned int numExtSems; /**< Number of handles and parameters supplied in extSemArray and paramsArray. */ +}; + +enum __device_builtin__ cudaGraphConditionalHandleFlags { + cudaGraphCondAssignDefault = 1 /**< Apply default handle value when graph is launched. */ +}; + +/** + * CUDA conditional node types + */ +enum __device_builtin__ cudaGraphConditionalNodeType { + cudaGraphCondTypeIf = 0, /**< Conditional 'if' Node. Body executed once if condition value is non-zero. */ + cudaGraphCondTypeWhile = 1, /**< Conditional 'while' Node. Body executed repeatedly while condition value is non-zero. */ +}; + +/** + * CUDA conditional node parameters + */ +struct __device_builtin__ cudaConditionalNodeParams { + cudaGraphConditionalHandle handle; /**< Conditional node handle. + Handles must be created in advance of creating the node + using ::cudaGraphConditionalHandleCreate. */ + enum cudaGraphConditionalNodeType type; /**< Type of conditional node. */ + unsigned int size; /**< Size of graph output array. Must be 1. */ + cudaGraph_t *phGraph_out; /**< CUDA-owned array populated with conditional node child graphs during creation of the node. + Valid for the lifetime of the conditional node. + The contents of the graph(s) are subject to the following constraints: + + - Allowed node types are kernel nodes, empty nodes, child graphs, memsets, + memcopies, and conditionals. This applies recursively to child graphs and conditional bodies. + - All kernels, including kernels in nested conditionals or child graphs at any level, + must belong to the same CUDA context. + + These graphs may be populated using graph node creation APIs or ::cudaStreamBeginCaptureToGraph. */ +}; + +/** +* CUDA Graph node types +*/ +enum __device_builtin__ cudaGraphNodeType { + cudaGraphNodeTypeKernel = 0x00, /**< GPU kernel node */ + cudaGraphNodeTypeMemcpy = 0x01, /**< Memcpy node */ + cudaGraphNodeTypeMemset = 0x02, /**< Memset node */ + cudaGraphNodeTypeHost = 0x03, /**< Host (executable) node */ + cudaGraphNodeTypeGraph = 0x04, /**< Node which executes an embedded graph */ + cudaGraphNodeTypeEmpty = 0x05, /**< Empty (no-op) node */ + cudaGraphNodeTypeWaitEvent = 0x06, /**< External event wait node */ + cudaGraphNodeTypeEventRecord = 0x07, /**< External event record node */ + cudaGraphNodeTypeExtSemaphoreSignal = 0x08, /**< External semaphore signal node */ + cudaGraphNodeTypeExtSemaphoreWait = 0x09, /**< External semaphore wait node */ + cudaGraphNodeTypeMemAlloc = 0x0a, /**< Memory allocation node */ + cudaGraphNodeTypeMemFree = 0x0b, /**< Memory free node */ + cudaGraphNodeTypeConditional = 0x0d, /**< Conditional node + + May be used to implement a conditional execution path or loop + inside of a graph. The graph(s) contained within the body of the conditional node + can be selectively executed or iterated upon based on the value of a conditional + variable. + + Handles must be created in advance of creating the node + using ::cudaGraphConditionalHandleCreate. + + The following restrictions apply to graphs which contain conditional nodes: + The graph cannot be used in a child node. + Only one instantiation of the graph may exist at any point in time. + The graph cannot be cloned. + + To set the control value, supply a default value when creating the handle and/or + call ::cudaGraphSetConditional from device code.*/ + cudaGraphNodeTypeCount +}; + +/** + * Child graph node parameters + */ +struct __device_builtin__ cudaChildGraphNodeParams { + cudaGraph_t graph; /**< The child graph to clone into the node for node creation, or + a handle to the graph owned by the node for node query */ +}; + +/** + * Event record node parameters + */ +struct __device_builtin__ cudaEventRecordNodeParams { + cudaEvent_t event; /**< The event to record when the node executes */ +}; + +/** + * Event wait node parameters + */ +struct __device_builtin__ cudaEventWaitNodeParams { + cudaEvent_t event; /**< The event to wait on from the node */ +}; + +/** + * Graph node parameters. See ::cudaGraphAddNode. + */ +struct __device_builtin__ cudaGraphNodeParams { + enum cudaGraphNodeType type; /**< Type of the node */ + int reserved0[3]; /**< Reserved. Must be zero. */ + + union { + long long reserved1[29]; /**< Padding. Unused bytes must be zero. */ + struct cudaKernelNodeParamsV2 kernel; /**< Kernel node parameters. */ + struct cudaMemcpyNodeParams memcpy; /**< Memcpy node parameters. */ + struct cudaMemsetParamsV2 memset; /**< Memset node parameters. */ + struct cudaHostNodeParamsV2 host; /**< Host node parameters. */ + struct cudaChildGraphNodeParams graph; /**< Child graph node parameters. */ + struct cudaEventWaitNodeParams eventWait; /**< Event wait node parameters. */ + struct cudaEventRecordNodeParams eventRecord; /**< Event record node parameters. */ + struct cudaExternalSemaphoreSignalNodeParamsV2 extSemSignal; /**< External semaphore signal node parameters. */ + struct cudaExternalSemaphoreWaitNodeParamsV2 extSemWait; /**< External semaphore wait node parameters. */ + struct cudaMemAllocNodeParamsV2 alloc; /**< Memory allocation node parameters. */ + struct cudaMemFreeNodeParams free; /**< Memory free node parameters. */ + struct cudaConditionalNodeParams conditional; /**< Conditional node parameters. */ + }; + + long long reserved2; /**< Reserved bytes. Must be zero. */ +}; + +/** + * Type annotations that can be applied to graph edges as part of ::cudaGraphEdgeData. + */ +typedef __device_builtin__ enum cudaGraphDependencyType_enum { + cudaGraphDependencyTypeDefault = 0, /**< This is an ordinary dependency. */ + cudaGraphDependencyTypeProgrammatic = 1 /**< This dependency type allows the downstream node to + use \c cudaGridDependencySynchronize(). It may only be used + between kernel nodes, and must be used with either the + ::cudaGraphKernelNodePortProgrammatic or + ::cudaGraphKernelNodePortLaunchCompletion outgoing port. */ +} cudaGraphDependencyType; + +/** + * Optional annotation for edges in a CUDA graph. Note, all edges implicitly have annotations and + * default to a zero-initialized value if not specified. A zero-initialized struct indicates a + * standard full serialization of two nodes with memory visibility. + */ +typedef __device_builtin__ struct cudaGraphEdgeData_st { + unsigned char from_port; /**< This indicates when the dependency is triggered from the upstream + node on the edge. The meaning is specfic to the node type. A value + of 0 in all cases means full completion of the upstream node, with + memory visibility to the downstream node or portion thereof + (indicated by \c to_port). +
+ Only kernel nodes define non-zero ports. A kernel node + can use the following output port types: + ::cudaGraphKernelNodePortDefault, ::cudaGraphKernelNodePortProgrammatic, + or ::cudaGraphKernelNodePortLaunchCompletion. */ + unsigned char to_port; /**< This indicates what portion of the downstream node is dependent on + the upstream node or portion thereof (indicated by \c from_port). The + meaning is specific to the node type. A value of 0 in all cases means + the entirety of the downstream node is dependent on the upstream work. +
+ Currently no node types define non-zero ports. Accordingly, this field + must be set to zero. */ + unsigned char type; /**< This should be populated with a value from ::cudaGraphDependencyType. (It + is typed as char due to compiler-specific layout of bitfields.) See + ::cudaGraphDependencyType. */ + unsigned char reserved[5]; /**< These bytes are unused and must be zeroed. This ensures + compatibility if additional fields are added in the future. */ +} cudaGraphEdgeData; + +/** + * This port activates when the kernel has finished executing. + */ +#define cudaGraphKernelNodePortDefault 0 +/** + * This port activates when all blocks of the kernel have performed cudaTriggerProgrammaticLaunchCompletion() + * or have terminated. It must be used with edge type ::cudaGraphDependencyTypeProgrammatic. See also + * ::cudaLaunchAttributeProgrammaticEvent. + */ +#define cudaGraphKernelNodePortProgrammatic 1 +/** + * This port activates when all blocks of the kernel have begun execution. See also + * ::cudaLaunchAttributeLaunchCompletionEvent. + */ +#define cudaGraphKernelNodePortLaunchCompletion 2 + +/** + * CUDA executable (launchable) graph + */ +typedef struct CUgraphExec_st* cudaGraphExec_t; + +/** +* CUDA Graph Update error types +*/ +enum __device_builtin__ cudaGraphExecUpdateResult { + cudaGraphExecUpdateSuccess = 0x0, /**< The update succeeded */ + cudaGraphExecUpdateError = 0x1, /**< The update failed for an unexpected reason which is described in the return value of the function */ + cudaGraphExecUpdateErrorTopologyChanged = 0x2, /**< The update failed because the topology changed */ + cudaGraphExecUpdateErrorNodeTypeChanged = 0x3, /**< The update failed because a node type changed */ + cudaGraphExecUpdateErrorFunctionChanged = 0x4, /**< The update failed because the function of a kernel node changed (CUDA driver < 11.2) */ + cudaGraphExecUpdateErrorParametersChanged = 0x5, /**< The update failed because the parameters changed in a way that is not supported */ + cudaGraphExecUpdateErrorNotSupported = 0x6, /**< The update failed because something about the node is not supported */ + cudaGraphExecUpdateErrorUnsupportedFunctionChange = 0x7, /**< The update failed because the function of a kernel node changed in an unsupported way */ + cudaGraphExecUpdateErrorAttributesChanged = 0x8 /**< The update failed because the node attributes changed in a way that is not supported */ +}; + +/** + * Graph instantiation results +*/ +typedef __device_builtin__ enum cudaGraphInstantiateResult { + cudaGraphInstantiateSuccess = 0, /**< Instantiation succeeded */ + cudaGraphInstantiateError = 1, /**< Instantiation failed for an unexpected reason which is described in the return value of the function */ + cudaGraphInstantiateInvalidStructure = 2, /**< Instantiation failed due to invalid structure, such as cycles */ + cudaGraphInstantiateNodeOperationNotSupported = 3, /**< Instantiation for device launch failed because the graph contained an unsupported operation */ + cudaGraphInstantiateMultipleDevicesNotSupported = 4 /**< Instantiation for device launch failed due to the nodes belonging to different contexts */ +} cudaGraphInstantiateResult; + +/** + * Graph instantiation parameters + */ +typedef __device_builtin__ struct cudaGraphInstantiateParams_st +{ + unsigned long long flags; /**< Instantiation flags */ + cudaStream_t uploadStream; /**< Upload stream */ + cudaGraphNode_t errNode_out; /**< The node which caused instantiation to fail, if any */ + cudaGraphInstantiateResult result_out; /**< Whether instantiation was successful. If it failed, the reason why */ +} cudaGraphInstantiateParams; + +/** + * Result information returned by cudaGraphExecUpdate + */ +typedef __device_builtin__ struct cudaGraphExecUpdateResultInfo_st { + /** + * Gives more specific detail when a cuda graph update fails. + */ + enum cudaGraphExecUpdateResult result; + + /** + * The "to node" of the error edge when the topologies do not match. + * The error node when the error is associated with a specific node. + * NULL when the error is generic. + */ + cudaGraphNode_t errorNode; + + /** + * The from node of error edge when the topologies do not match. Otherwise NULL. + */ + cudaGraphNode_t errorFromNode; +} cudaGraphExecUpdateResultInfo; + +/** + * CUDA device node handle for device-side node update + */ +typedef struct CUgraphDeviceUpdatableNode_st* cudaGraphDeviceNode_t; + +/** + * Specifies the field to update when performing multiple node updates from the device + */ +enum __device_builtin__ cudaGraphKernelNodeField +{ + cudaGraphKernelNodeFieldInvalid = 0, /**< Invalid field */ + cudaGraphKernelNodeFieldGridDim, /**< Grid dimension update */ + cudaGraphKernelNodeFieldParam, /**< Kernel parameter update */ + cudaGraphKernelNodeFieldEnabled /**< Node enable/disable */ +}; + +/** + * Struct to specify a single node update to pass as part of a larger array to ::cudaGraphKernelNodeUpdatesApply + */ +struct __device_builtin__ cudaGraphKernelNodeUpdate { + cudaGraphDeviceNode_t node; /**< Node to update */ + enum cudaGraphKernelNodeField field; /**< Which type of update to apply. Determines how updateData is interpreted */ + union { +#if !defined(__cplusplus) || __cplusplus >= 201103L + dim3 gridDim; /**< Grid dimensions */ +#else + /* Union members cannot have nontrivial constructors until C++11. */ + uint3 gridDim; /**< Grid dimensions */ +#endif + struct { + const void *pValue; /**< Kernel parameter data to write in */ + size_t offset; /**< Offset into the parameter buffer at which to apply the update */ + size_t size; /**< Number of bytes to update */ + } param; /**< Kernel parameter data */ + unsigned int isEnabled; /**< Node enable/disable data. Nonzero if the node should be enabled, 0 if it should be disabled */ + } updateData; /**< Update data to apply. Which field is used depends on field's value */ +}; + +/** + * Flags to specify search options to be used with ::cudaGetDriverEntryPoint + * For more details see ::cuGetProcAddress + */ +enum __device_builtin__ cudaGetDriverEntryPointFlags { + cudaEnableDefault = 0x0, /**< Default search mode for driver symbols. */ + cudaEnableLegacyStream = 0x1, /**< Search for legacy versions of driver symbols. */ + cudaEnablePerThreadDefaultStream = 0x2 /**< Search for per-thread versions of driver symbols. */ +}; + +/** + * Enum for status from obtaining driver entry points, used with ::cudaApiGetDriverEntryPoint + */ +enum __device_builtin__ cudaDriverEntryPointQueryResult { + cudaDriverEntryPointSuccess = 0, /**< Search for symbol found a match */ + cudaDriverEntryPointSymbolNotFound = 1, /**< Search for symbol was not found */ + cudaDriverEntryPointVersionNotSufficent = 2 /**< Search for symbol was found but version wasn't great enough */ +}; + +/** + * CUDA Graph debug write options + */ +enum __device_builtin__ cudaGraphDebugDotFlags { + cudaGraphDebugDotFlagsVerbose = 1<<0, /**< Output all debug data as if every debug flag is enabled */ + cudaGraphDebugDotFlagsKernelNodeParams = 1<<2, /**< Adds cudaKernelNodeParams to output */ + cudaGraphDebugDotFlagsMemcpyNodeParams = 1<<3, /**< Adds cudaMemcpy3DParms to output */ + cudaGraphDebugDotFlagsMemsetNodeParams = 1<<4, /**< Adds cudaMemsetParams to output */ + cudaGraphDebugDotFlagsHostNodeParams = 1<<5, /**< Adds cudaHostNodeParams to output */ + cudaGraphDebugDotFlagsEventNodeParams = 1<<6, /**< Adds cudaEvent_t handle from record and wait nodes to output */ + cudaGraphDebugDotFlagsExtSemasSignalNodeParams = 1<<7, /**< Adds cudaExternalSemaphoreSignalNodeParams values to output */ + cudaGraphDebugDotFlagsExtSemasWaitNodeParams = 1<<8, /**< Adds cudaExternalSemaphoreWaitNodeParams to output */ + cudaGraphDebugDotFlagsKernelNodeAttributes = 1<<9, /**< Adds cudaKernelNodeAttrID values to output */ + cudaGraphDebugDotFlagsHandles = 1<<10 /**< Adds node handles and every kernel function handle to output */ + ,cudaGraphDebugDotFlagsConditionalNodeParams = 1<<15, /**< Adds cudaConditionalNodeParams to output */ +}; + +/** + * Flags for instantiating a graph + */ +enum __device_builtin__ cudaGraphInstantiateFlags { + cudaGraphInstantiateFlagAutoFreeOnLaunch = 1 /**< Automatically free memory allocated in a graph before relaunching. */ + , cudaGraphInstantiateFlagUpload = 2 /**< Automatically upload the graph after instantiation. Only supported by + ::cudaGraphInstantiateWithParams. The upload will be performed using the + stream provided in \p instantiateParams. */ + , cudaGraphInstantiateFlagDeviceLaunch = 4 /**< Instantiate the graph to be launchable from the device. This flag can only + be used on platforms which support unified addressing. This flag cannot be + used in conjunction with cudaGraphInstantiateFlagAutoFreeOnLaunch. */ + , cudaGraphInstantiateFlagUseNodePriority = 8 /**< Run the graph using the per-node priority attributes rather than the + priority of the stream it is launched into. */ +}; + +/** + * Memory Synchronization Domain + * + * A kernel can be launched in a specified memory synchronization domain that affects all memory operations issued by + * that kernel. A memory barrier issued in one domain will only order memory operations in that domain, thus eliminating + * latency increase from memory barriers ordering unrelated traffic. + * + * By default, kernels are launched in domain 0. Kernel launched with ::cudaLaunchMemSyncDomainRemote will have a + * different domain ID. User may also alter the domain ID with ::cudaLaunchMemSyncDomainMap for a specific stream / + * graph node / kernel launch. See ::cudaLaunchAttributeMemSyncDomain, ::cudaStreamSetAttribute, ::cudaLaunchKernelEx, + * ::cudaGraphKernelNodeSetAttribute. + * + * Memory operations done in kernels launched in different domains are considered system-scope distanced. In other + * words, a GPU scoped memory synchronization is not sufficient for memory order to be observed by kernels in another + * memory synchronization domain even if they are on the same GPU. + */ +typedef __device_builtin__ enum cudaLaunchMemSyncDomain { + cudaLaunchMemSyncDomainDefault = 0, /**< Launch kernels in the default domain */ + cudaLaunchMemSyncDomainRemote = 1 /**< Launch kernels in the remote domain */ +} cudaLaunchMemSyncDomain; + +/** + * Memory Synchronization Domain map + * + * See ::cudaLaunchMemSyncDomain. + * + * By default, kernels are launched in domain 0. Kernel launched with ::cudaLaunchMemSyncDomainRemote will have a + * different domain ID. User may also alter the domain ID with ::cudaLaunchMemSyncDomainMap for a specific stream / + * graph node / kernel launch. See ::cudaLaunchAttributeMemSyncDomainMap. + * + * Domain ID range is available through ::cudaDevAttrMemSyncDomainCount. + */ +typedef __device_builtin__ struct cudaLaunchMemSyncDomainMap_st { + unsigned char default_; /**< The default domain ID to use for designated kernels */ + unsigned char remote; /**< The remote domain ID to use for designated kernels */ +} cudaLaunchMemSyncDomainMap; + +/** + * Launch attributes enum; used as id field of ::cudaLaunchAttribute + */ +typedef __device_builtin__ enum cudaLaunchAttributeID { + cudaLaunchAttributeIgnore = 0 /**< Ignored entry, for convenient composition */ + , cudaLaunchAttributeAccessPolicyWindow = 1 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::accessPolicyWindow. */ + , cudaLaunchAttributeCooperative = 2 /**< Valid for graph nodes, launches. See + ::cudaLaunchAttributeValue::cooperative. */ + , cudaLaunchAttributeSynchronizationPolicy = 3 /**< Valid for streams. See ::cudaLaunchAttributeValue::syncPolicy. */ + , cudaLaunchAttributeClusterDimension = 4 /**< Valid for graph nodes, launches. See + ::cudaLaunchAttributeValue::clusterDim. */ + , cudaLaunchAttributeClusterSchedulingPolicyPreference = 5 /**< Valid for graph nodes, launches. See + ::cudaLaunchAttributeValue::clusterSchedulingPolicyPreference. */ + , cudaLaunchAttributeProgrammaticStreamSerialization = 6 /**< Valid for launches. Setting + ::cudaLaunchAttributeValue::programmaticStreamSerializationAllowed + to non-0 signals that the kernel will use programmatic + means to resolve its stream dependency, so that the + CUDA runtime should opportunistically allow the grid's + execution to overlap with the previous kernel in the + stream, if that kernel requests the overlap. The + dependent launches can choose to wait on the + dependency using the programmatic sync + (cudaGridDependencySynchronize() or equivalent PTX + instructions). */ + , cudaLaunchAttributeProgrammaticEvent = 7 /**< Valid for launches. Set + ::cudaLaunchAttributeValue::programmaticEvent to + record the event. Event recorded through this launch + attribute is guaranteed to only trigger after all + block in the associated kernel trigger the event. A + block can trigger the event programmatically in a + future CUDA release. A trigger can also be inserted at + the beginning of each block's execution if + triggerAtBlockStart is set to non-0. The dependent + launches can choose to wait on the dependency using + the programmatic sync (cudaGridDependencySynchronize() + or equivalent PTX instructions). Note that dependents + (including the CPU thread calling + cudaEventSynchronize()) are not guaranteed to observe + the release precisely when it is released. For + example, cudaEventSynchronize() may only observe the + event trigger long after the associated kernel has + completed. This recording type is primarily meant for + establishing programmatic dependency between device + tasks. Note also this type of dependency allows, but + does not guarantee, concurrent execution of tasks. +
+ The event supplied must not be an interprocess or + interop event. The event must disable timing (i.e. + must be created with the ::cudaEventDisableTiming flag + set). */ + , cudaLaunchAttributePriority = 8 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::priority. */ + , cudaLaunchAttributeMemSyncDomainMap = 9 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::memSyncDomainMap. */ + , cudaLaunchAttributeMemSyncDomain = 10 /**< Valid for streams, graph nodes, launches. See + ::cudaLaunchAttributeValue::memSyncDomain. */ + , cudaLaunchAttributeLaunchCompletionEvent = 12 /**< Valid for launches. Set + ::cudaLaunchAttributeValue::launchCompletionEvent to record the + event. +
+ Nominally, the event is triggered once all blocks of the kernel + have begun execution. Currently this is a best effort. If a kernel + B has a launch completion dependency on a kernel A, B may wait + until A is complete. Alternatively, blocks of B may begin before + all blocks of A have begun, for example if B can claim execution + resources unavailable to A (e.g. they run on different GPUs) or + if B is a higher priority than A. + Exercise caution if such an ordering inversion could lead + to deadlock. +
+ A launch completion event is nominally similar to a programmatic + event with \c triggerAtBlockStart set except that it is not + visible to \c cudaGridDependencySynchronize() and can be used with + compute capability less than 9.0. +
+ The event supplied must not be an interprocess or interop event. + The event must disable timing (i.e. must be created with the + ::cudaEventDisableTiming flag set). */ + , cudaLaunchAttributeDeviceUpdatableKernelNode = 13 /**< Valid for graph nodes, launches. This attribute is graphs-only, + and passing it to a launch in a non-capturing stream will result + in an error. +
+ :cudaLaunchAttributeValue::deviceUpdatableKernelNode::deviceUpdatable can + only be set to 0 or 1. Setting the field to 1 indicates that the + corresponding kernel node should be device-updatable. On success, a handle + will be returned via + ::cudaLaunchAttributeValue::deviceUpdatableKernelNode::devNode which can be + passed to the various device-side update functions to update the node's + kernel parameters from within another kernel. For more information on the + types of device updates that can be made, as well as the relevant limitations + thereof, see ::cudaGraphKernelNodeUpdatesApply. +
+ Nodes which are device-updatable have additional restrictions compared to + regular kernel nodes. Firstly, device-updatable nodes cannot be removed + from their graph via ::cudaGraphDestroyNode. Additionally, once opted-in + to this functionality, a node cannot opt out, and any attempt to set the + deviceUpdatable attribute to 0 will result in an error. Device-updatable + kernel nodes also cannot have their attributes copied to/from another kernel + node via ::cudaGraphKernelNodeCopyAttributes. Graphs containing one or more + device-updatable nodes also do not allow multiple instantiation, and neither + the graph nor its instantiated version can be passed to ::cudaGraphExecUpdate. +
+ If a graph contains device-updatable nodes and updates those nodes from the device + from within the graph, the graph must be uploaded with ::cuGraphUpload before it + is launched. For such a graph, if host-side executable graph updates are made to the + device-updatable nodes, the graph must be uploaded before it is launched again. */ +} cudaLaunchAttributeID; + +/** + * Launch attributes union; used as value field of ::cudaLaunchAttribute + */ +typedef __device_builtin__ union cudaLaunchAttributeValue { + char pad[64]; /* Pad to 64 bytes */ + struct cudaAccessPolicyWindow accessPolicyWindow; /**< Value of launch attribute ::cudaLaunchAttributeAccessPolicyWindow. */ + int cooperative; /**< Value of launch attribute ::cudaLaunchAttributeCooperative. Nonzero indicates a cooperative + kernel (see ::cudaLaunchCooperativeKernel). */ + enum cudaSynchronizationPolicy syncPolicy; /**< Value of launch attribute + ::cudaLaunchAttributeSynchronizationPolicy. ::cudaSynchronizationPolicy + for work queued up in this stream. */ + /** + * Value of launch attribute ::cudaLaunchAttributeClusterDimension that + * represents the desired cluster dimensions for the kernel. Opaque type + * with the following fields: + * - \p x - The X dimension of the cluster, in blocks. Must be a divisor + * of the grid X dimension. + * - \p y - The Y dimension of the cluster, in blocks. Must be a divisor + * of the grid Y dimension. + * - \p z - The Z dimension of the cluster, in blocks. Must be a divisor + * of the grid Z dimension. + */ + struct { + unsigned int x; + unsigned int y; + unsigned int z; + } clusterDim; + enum cudaClusterSchedulingPolicy clusterSchedulingPolicyPreference; /**< Value of launch attribute + ::cudaLaunchAttributeClusterSchedulingPolicyPreference. Cluster + scheduling policy preference for the kernel. */ + int programmaticStreamSerializationAllowed; /**< Value of launch attribute + ::cudaLaunchAttributeProgrammaticStreamSerialization. */ + struct { + cudaEvent_t event; /**< Event to fire when all blocks trigger it */ + int flags; /**< Event record flags, see ::cudaEventRecordWithFlags. Does not accept + ::cudaEventRecordExternal. */ + int triggerAtBlockStart; /**< If this is set to non-0, each block launch will automatically trigger the event */ + } programmaticEvent; /**< Value of launch attribute ::cudaLaunchAttributeProgrammaticEvent. */ + int priority; /**< Value of launch attribute ::cudaLaunchAttributePriority. Execution priority of the kernel. */ + cudaLaunchMemSyncDomainMap memSyncDomainMap; /**< Value of launch attribute + ::cudaLaunchAttributeMemSyncDomainMap. See + ::cudaLaunchMemSyncDomainMap. */ + cudaLaunchMemSyncDomain memSyncDomain; /**< Value of launch attribute ::cudaLaunchAttributeMemSyncDomain. See + ::cudaLaunchMemSyncDomain. */ + struct { + cudaEvent_t event; /**< Event to fire when the last block launches */ + int flags; /**< Event record flags, see ::cudaEventRecordWithFlags. Does not accept + ::cudaEventRecordExternal. */ + } launchCompletionEvent; /**< Value of launch attribute ::cudaLaunchAttributeLaunchCompletionEvent. */ + + struct { + int deviceUpdatable; /**< Whether or not the resulting kernel node should be device-updatable. */ + cudaGraphDeviceNode_t devNode; /**< Returns a handle to pass to the various device-side update functions. */ + } deviceUpdatableKernelNode; /**< Value of launch attribute ::CU_LAUNCH_ATTRIBUTE_DEVICE_UPDATABLE_KERNEL_NODE. */ +} cudaLaunchAttributeValue; + +/** + * Launch attribute + */ +typedef __device_builtin__ struct cudaLaunchAttribute_st { + cudaLaunchAttributeID id; /**< Attribute to set */ + char pad[8 - sizeof(cudaLaunchAttributeID)]; + cudaLaunchAttributeValue val; /**< Value of the attribute */ +} cudaLaunchAttribute; + +/** + * CUDA extensible launch configuration + */ +typedef __device_builtin__ struct cudaLaunchConfig_st { + dim3 gridDim; /**< Grid dimensions */ + dim3 blockDim; /**< Block dimensions */ + size_t dynamicSmemBytes; /**< Dynamic shared-memory size per thread block in bytes */ + cudaStream_t stream; /**< Stream identifier */ + cudaLaunchAttribute *attrs; /**< List of attributes; nullable if ::cudaLaunchConfig_t::numAttrs == 0 */ + unsigned int numAttrs; /**< Number of attributes populated in ::cudaLaunchConfig_t::attrs */ +} cudaLaunchConfig_t; + +#define cudaStreamAttrID cudaLaunchAttributeID +#define cudaStreamAttributeAccessPolicyWindow cudaLaunchAttributeAccessPolicyWindow +#define cudaStreamAttributeSynchronizationPolicy cudaLaunchAttributeSynchronizationPolicy +#define cudaStreamAttributeMemSyncDomainMap cudaLaunchAttributeMemSyncDomainMap +#define cudaStreamAttributeMemSyncDomain cudaLaunchAttributeMemSyncDomain +#define cudaStreamAttributePriority cudaLaunchAttributePriority + +#define cudaStreamAttrValue cudaLaunchAttributeValue + +#define cudaKernelNodeAttrID cudaLaunchAttributeID +#define cudaKernelNodeAttributeAccessPolicyWindow cudaLaunchAttributeAccessPolicyWindow +#define cudaKernelNodeAttributeCooperative cudaLaunchAttributeCooperative +#define cudaKernelNodeAttributePriority cudaLaunchAttributePriority +#define cudaKernelNodeAttributeClusterDimension cudaLaunchAttributeClusterDimension +#define cudaKernelNodeAttributeClusterSchedulingPolicyPreference cudaLaunchAttributeClusterSchedulingPolicyPreference +#define cudaKernelNodeAttributeMemSyncDomainMap cudaLaunchAttributeMemSyncDomainMap +#define cudaKernelNodeAttributeMemSyncDomain cudaLaunchAttributeMemSyncDomain +#define cudaKernelNodeAttributeDeviceUpdatableKernelNode cudaLaunchAttributeDeviceUpdatableKernelNode + +#define cudaKernelNodeAttrValue cudaLaunchAttributeValue + +enum __device_builtin__ cudaDeviceNumaConfig { + cudaDeviceNumaConfigNone = 0, /**< The GPU is not a NUMA node */ + cudaDeviceNumaConfigNumaNode, /**< The GPU is a NUMA node, cudaDevAttrNumaId contains its NUMA ID */ +}; + +/** + * CUDA async callback handle + */ +typedef struct cudaAsyncCallbackEntry* cudaAsyncCallbackHandle_t; + +struct cudaAsyncCallbackEntry; + +/** +* Types of async notification that can occur +*/ +typedef __device_builtin__ enum cudaAsyncNotificationType_enum { + cudaAsyncNotificationTypeOverBudget = 0x1 +} cudaAsyncNotificationType; + +/** +* Information describing an async notification event +*/ +typedef __device_builtin__ struct cudaAsyncNotificationInfo +{ + cudaAsyncNotificationType type; + union { + struct { + unsigned long long bytesOverBudget; + } overBudget; + } info; +} cudaAsyncNotificationInfo_t; + +typedef void (*cudaAsyncCallback)(cudaAsyncNotificationInfo_t*, void*, cudaAsyncCallbackHandle_t); + + +/** @} */ +/** @} */ /* END CUDART_TYPES */ + +#endif /* !__CUDACC_RTC_MINIMAL__ */ + +#if defined(__UNDEF_CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS_DRIVER_TYPES_H__) +#undef __CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS__ +#undef __UNDEF_CUDA_INCLUDE_COMPILER_INTERNAL_HEADERS_DRIVER_TYPES_H__ +#endif + +#undef __CUDA_DEPRECATED + + + +#endif /* !__DRIVER_TYPES_H__ */ diff --git a/lib/python3.12/site-packages/triton/backends/nvidia/include/sm_32_atomic_functions.hpp b/lib/python3.12/site-packages/triton/backends/nvidia/include/sm_32_atomic_functions.hpp new file mode 100644 index 0000000000000000000000000000000000000000..7cfea072a044ceac7fb96f3bf3006520a108020e --- /dev/null +++ b/lib/python3.12/site-packages/triton/backends/nvidia/include/sm_32_atomic_functions.hpp @@ -0,0 +1,151 @@ +/* + * Copyright 1993-2023 NVIDIA Corporation. All rights reserved. + * + * NOTICE TO LICENSEE: + * + * This source code and/or documentation ("Licensed Deliverables") are + * subject to NVIDIA intellectual property rights under U.S. and + * international Copyright laws. + * + * These Licensed Deliverables contained herein is PROPRIETARY and + * CONFIDENTIAL to NVIDIA and is being provided under the terms and + * conditions of a form of NVIDIA software license agreement by and + * between NVIDIA and Licensee ("License Agreement") or electronically + * accepted by Licensee. Notwithstanding any terms or conditions to + * the contrary in the License Agreement, reproduction or disclosure + * of the Licensed Deliverables to any third party without the express + * written consent of NVIDIA is prohibited. + * + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, NVIDIA MAKES NO REPRESENTATION ABOUT THE + * SUITABILITY OF THESE LICENSED DELIVERABLES FOR ANY PURPOSE. IT IS + * PROVIDED "AS IS" WITHOUT EXPRESS OR IMPLIED WARRANTY OF ANY KIND. + * NVIDIA DISCLAIMS ALL WARRANTIES WITH REGARD TO THESE LICENSED + * DELIVERABLES, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY, + * NONINFRINGEMENT, AND FITNESS FOR A PARTICULAR PURPOSE. + * NOTWITHSTANDING ANY TERMS OR CONDITIONS TO THE CONTRARY IN THE + * LICENSE AGREEMENT, IN NO EVENT SHALL NVIDIA BE LIABLE FOR ANY + * SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, OR ANY + * DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, + * WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS + * ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE + * OF THESE LICENSED DELIVERABLES. + * + * U.S. Government End Users. These Licensed Deliverables are a + * "commercial item" as that term is defined at 48 C.F.R. 2.101 (OCT + * 1995), consisting of "commercial computer software" and "commercial + * computer software documentation" as such terms are used in 48 + * C.F.R. 35.235 (SEPT 1995) and is provided to the U.S. Government + * only as a commercial end item. Consistent with 48 C.F.R.35.235 and + * 48 C.F.R. 227.7202-1 through 227.7202-4 (JUNE 1995), all + * U.S. Government End Users acquire the Licensed Deliverables with + * only those rights set forth herein. + * + * Any use of the Licensed Deliverables in individual and commercial + * software must include, in the user documentation and internal + * comments to the code, the above Disclaimer and U.S. Government End + * Users Notice. + */ + +#if !defined(__SM_32_ATOMIC_FUNCTIONS_HPP__) +#define __SM_32_ATOMIC_FUNCTIONS_HPP__ + +#ifdef __CUDA_ARCH__ +extern "C" +{ +extern __device__ __device_builtin__ long long __illAtomicMin(long long *address, long long val); +extern __device__ __device_builtin__ long long __illAtomicMax(long long *address, long long val); +extern __device__ __device_builtin__ long long __llAtomicAnd(long long *address, long long val); +extern __device__ __device_builtin__ long long __llAtomicOr(long long *address, long long val); +extern __device__ __device_builtin__ long long __llAtomicXor(long long *address, long long val); +extern __device__ __device_builtin__ unsigned long long __ullAtomicMin(unsigned long long *address, unsigned long long val); +extern __device__ __device_builtin__ unsigned long long __ullAtomicMax(unsigned long long *address, unsigned long long val); +extern __device__ __device_builtin__ unsigned long long __ullAtomicAnd(unsigned long long *address, unsigned long long val); +extern __device__ __device_builtin__ unsigned long long __ullAtomicOr (unsigned long long *address, unsigned long long val); +extern __device__ __device_builtin__ unsigned long long __ullAtomicXor(unsigned long long *address, unsigned long long val); +} +#endif /* __CUDA_ARCH__ */ + + +#if defined(__CUDACC_RTC__) +#define __SM_32_ATOMIC_FUNCTIONS_DECL__ __device__ +#else /* !__CUDACC_RTC__ */ +#define __SM_32_ATOMIC_FUNCTIONS_DECL__ static __inline__ __device__ +#endif /* __CUDACC_RTC__ */ + +#if defined(__cplusplus) && defined(__CUDACC__) + +#if defined(_NVHPC_CUDA) || !defined(__CUDA_ARCH__) || __CUDA_ARCH__ >= 320 + +/******************************************************************************* +* * +* * +* * +*******************************************************************************/ + +#include "cuda_runtime_api.h" + +/******************************************************************************* +* * +* * +* * +*******************************************************************************/ + +__SM_32_ATOMIC_FUNCTIONS_DECL__ long long atomicMin(long long *address, long long val) +{ + return __illAtomicMin(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ long long atomicMax(long long *address, long long val) +{ + return __illAtomicMax(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ long long atomicAnd(long long *address, long long val) +{ + return __llAtomicAnd(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ long long atomicOr(long long *address, long long val) +{ + return __llAtomicOr(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ long long atomicXor(long long *address, long long val) +{ + return __llAtomicXor(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ unsigned long long atomicMin(unsigned long long *address, unsigned long long val) +{ + return __ullAtomicMin(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ unsigned long long atomicMax(unsigned long long *address, unsigned long long val) +{ + return __ullAtomicMax(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ unsigned long long atomicAnd(unsigned long long *address, unsigned long long val) +{ + return __ullAtomicAnd(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ unsigned long long atomicOr(unsigned long long *address, unsigned long long val) +{ + return __ullAtomicOr(address, val); +} + +__SM_32_ATOMIC_FUNCTIONS_DECL__ unsigned long long atomicXor(unsigned long long *address, unsigned long long val) +{ + return __ullAtomicXor(address, val); +} + +#endif /* _NVHPC_CUDA || !__CUDA_ARCH__ || __CUDA_ARCH__ >= 320 */ + +#endif /* __cplusplus && __CUDACC__ */ + +#undef __SM_32_ATOMIC_FUNCTIONS_DECL__ + +#endif /* !__SM_32_ATOMIC_FUNCTIONS_HPP__ */ + diff --git a/lib/python3.12/site-packages/triton/errors.py b/lib/python3.12/site-packages/triton/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..3a0a863553b9d62ccb60d0090ee6d66f8b9e3b79 --- /dev/null +++ b/lib/python3.12/site-packages/triton/errors.py @@ -0,0 +1,5 @@ +"""Base class for all errors raised by Triton""" + + +class TritonError(Exception): + ... diff --git a/lib/python3.12/site-packages/triton/language/__init__.py b/lib/python3.12/site-packages/triton/language/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6502a5348f3ed59e3ee602c07f319add0651f6b7 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/__init__.py @@ -0,0 +1,294 @@ +"""isort:skip_file""" +# Import order is significant here. + +from . import math +from . import extra +from .standard import ( + argmax, + argmin, + cdiv, + cumprod, + cumsum, + flip, + interleave, + max, + min, + ravel, + sigmoid, + softmax, + sort, + sum, + swizzle2d, + xor_sum, + zeros, + zeros_like, +) +from .core import ( + PropagateNan, + TRITON_MAX_TENSOR_NUMEL, + _experimental_descriptor_load, + _experimental_descriptor_store, + add, + advance, + arange, + associative_scan, + assume, + atomic_add, + atomic_and, + atomic_cas, + atomic_max, + atomic_min, + atomic_or, + atomic_xchg, + atomic_xor, + bfloat16, + block_type, + broadcast, + broadcast_to, + cat, + cast, + clamp, + const, + constexpr, + debug_barrier, + device_assert, + device_print, + dot, + dot_scaled, + dtype, + expand_dims, + float16, + float32, + float64, + float8e4b15, + float8e4nv, + float8e4b8, + float8e5, + float8e5b16, + full, + function_type, + histogram, + inline_asm_elementwise, + int1, + int16, + int32, + int64, + int8, + join, + load, + make_block_ptr, + max_constancy, + max_contiguous, + maximum, + minimum, + multiple_of, + num_programs, + permute, + pi32_t, + pointer_type, + nv_tma_desc_type, + program_id, + range, + reduce, + reshape, + split, + static_assert, + static_print, + static_range, + store, + tensor, + trans, + uint16, + uint32, + uint64, + uint8, + view, + void, + where, +) +from .math import (umulhi, exp, exp2, fma, log, log2, cos, rsqrt, sin, sqrt, sqrt_rn, abs, fdiv, div_rn, erf, floor, + ceil) +from .random import ( + pair_uniform_to_normal, + philox, + philox_impl, + rand, + rand4x, + randint, + randint4x, + randn, + randn4x, + uint_to_uniform_float, +) + +__all__ = [ + "PropagateNan", + "TRITON_MAX_TENSOR_NUMEL", + "_experimental_descriptor_load", + "_experimental_descriptor_store", + "abs", + "add", + "advance", + "arange", + "argmax", + "argmin", + "associative_scan", + "assume", + "atomic_add", + "atomic_and", + "atomic_cas", + "atomic_max", + "atomic_min", + "atomic_or", + "atomic_xchg", + "atomic_xor", + "bfloat16", + "block_type", + "broadcast", + "broadcast_to", + "builtin", + "cat", + "cast", + "cdiv", + "ceil", + "clamp", + "const", + "constexpr", + "cos", + "cumprod", + "cumsum", + "debug_barrier", + "device_assert", + "device_print", + "div_rn", + "dot", + "dot_scaled", + "dtype", + "erf", + "exp", + "exp2", + "expand_dims", + "extra", + "fdiv", + "flip", + "float16", + "float32", + "float64", + "float8e4b15", + "float8e4nv", + "float8e4b8", + "float8e5", + "float8e5b16", + "floor", + "fma", + "full", + "function_type", + "histogram", + "inline_asm_elementwise", + "interleave", + "int1", + "int16", + "int32", + "int64", + "int8", + "ir", + "join", + "load", + "log", + "log2", + "make_block_ptr", + "math", + "max", + "max_constancy", + "max_contiguous", + "maximum", + "min", + "minimum", + "multiple_of", + "num_programs", + "pair_uniform_to_normal", + "permute", + "philox", + "philox_impl", + "pi32_t", + "pointer_type", + "nv_tma_desc_type", + "program_id", + "rand", + "rand4x", + "randint", + "randint4x", + "randn", + "randn4x", + "range", + "ravel", + "reduce", + "reshape", + "rsqrt", + "sigmoid", + "sin", + "softmax", + "sort", + "split", + "sqrt", + "sqrt_rn", + "static_assert", + "static_print", + "static_range", + "store", + "sum", + "swizzle2d", + "tensor", + "trans", + "triton", + "uint16", + "uint32", + "uint64", + "uint8", + "uint_to_uniform_float", + "umulhi", + "view", + "void", + "where", + "xor_sum", + "zeros", + "zeros_like", +] + + +def str_to_ty(name): + if name[0] == "*": + name = name[1:] + const = False + if name[0] == "k": + name = name[1:] + const = True + ty = str_to_ty(name) + return pointer_type(element_ty=ty, const=const) + + if name == "nvTmaDesc": + return nv_tma_desc_type() + + tys = { + "fp8e4nv": float8e4nv, + "fp8e4b8": float8e4b8, + "fp8e5": float8e5, + "fp8e5b16": float8e5b16, + "fp8e4b15": float8e4b15, + "fp16": float16, + "bf16": bfloat16, + "fp32": float32, + "fp64": float64, + "i1": int1, + "i8": int8, + "i16": int16, + "i32": int32, + "i64": int64, + "u1": int1, + "u8": uint8, + "u16": uint16, + "u32": uint32, + "u64": uint64, + "B": int1, + } + return tys[name] diff --git a/lib/python3.12/site-packages/triton/language/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..bf129000b229badf67b641fa6ace53d90e75f55f Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/__pycache__/_utils.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/__pycache__/_utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f014ad190d8bb7cc3a2a14ed04e4f7b454de906d Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/__pycache__/_utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/__pycache__/math.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/__pycache__/math.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ef8bdce949ada01dc35c2213fab446b2d7b6d4c4 Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/__pycache__/math.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/__pycache__/random.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/__pycache__/random.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..085ee4f02ff531369fcb628c437ae864876f58da Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/__pycache__/random.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/__pycache__/standard.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/__pycache__/standard.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..580dee6e71e4f4394c84232ad645960926d0ce53 Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/__pycache__/standard.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/_utils.py b/lib/python3.12/site-packages/triton/language/_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..b9aa69071674bd7030e8fcf57e24212d73832c7b --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/_utils.py @@ -0,0 +1,21 @@ +from typing import List + +TRITON_MAX_TENSOR_NUMEL = 1048576 + + +def is_power_of_two(x): + return (x & (x - 1)) == 0 + + +def validate_block_shape(shape: List[int]): + numel = 1 + for i, d in enumerate(shape): + if not isinstance(d, int): + raise TypeError(f"Shape element {i} must have type `constexpr[int]`, got `constexpr[{type(d)}]") + if not is_power_of_two(d): + raise ValueError(f"Shape element {i} must be a power of 2") + numel *= d + + if numel > TRITON_MAX_TENSOR_NUMEL: + raise ValueError(f"numel ({numel}) exceeds triton maximum tensor numel ({TRITON_MAX_TENSOR_NUMEL})") + return numel diff --git a/lib/python3.12/site-packages/triton/language/core.py b/lib/python3.12/site-packages/triton/language/core.py new file mode 100644 index 0000000000000000000000000000000000000000..e2c57b388bb0c1b2e84533271021f6ea301528d6 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/core.py @@ -0,0 +1,2694 @@ +from __future__ import annotations + +from warnings import warn +from contextlib import contextmanager +from enum import Enum +from functools import partial, wraps +import typing +from typing import Union, Callable, List, Sequence, TypeVar, Optional +import builtins +from ..runtime.jit import jit +import inspect +import os + +from .._C.libtriton import ir +from . import semantic +from ._utils import TRITON_MAX_TENSOR_NUMEL, validate_block_shape + +T = TypeVar('T') + +TRITON_BUILTIN = "__triton_builtin__" + +PropagateNan = ir.PROPAGATE_NAN + + +def builtin(fn: T) -> T: + """Mark a function as a builtin.""" + assert callable(fn) + + @wraps(fn) + def wrapper(*args, **kwargs): + if "_builder" not in kwargs or kwargs["_builder"] is None: + print(kwargs) + raise ValueError("Did you forget to add @triton.jit ? " + "(`_builder` argument must be provided outside of JIT functions.)") + return fn(*args, **kwargs) + + setattr(wrapper, TRITON_BUILTIN, True) + + return wrapper + + +def _tensor_member_fn(fn: T) -> T: + """Decorator that adds this free function as a member fn on class tensor. + + When called as a member function on class tensor, the first argument to `fn` + is `self`, i.e. the tensor object. + + If there are multiple decorators on a function, you probably want this one + to be the highest one (i.e. furthest from the function's `def`), so it's + applied last. + + Unfortunately you still need to add a type stub to the body of class tensor + in order for pytype to know about it. + """ + assert callable(fn) + orig_sig = inspect.signature(fn) + # Does fn take args other than _builder, _generator, and the tensor itself? + has_args = len(orig_sig.parameters.keys() - {"_builder", "_generator"}) > 1 + + if not fn.__doc__: + fn.__doc__ = "" + fn.__doc__ += f""" + This function can also be called as a member function on :py:class:`tensor`, + as :code:`x.{fn.__name__}({"..." if has_args else ""})` instead of + :code:`{fn.__name__}(x{", ..." if has_args else ""})`. + """ + + def wrapper(*args, **kwargs): + return fn(*args, **kwargs) + + # Match the signature of `fn`, but change the first arg to `self` so the + # docs are a little less weird. + new_params = list(orig_sig.parameters.values()) + new_params[0] = new_params[0].replace(name='self') + new_sig = orig_sig.replace(parameters=new_params) + wrapper.__signature__ = new_sig + wrapper.__doc__ = f"Forwards to :py:func:`{fn.__name__}` free function" + # If fn is a builtin, mark the wrapper as a builtin too. + if is_builtin(fn): + setattr(wrapper, TRITON_BUILTIN, True) + + setattr(tensor, fn.__name__, wrapper) + return fn + + +def _unwrap_iterable(x): + """Returns x[0] if x has one element and x[0] is iterable.""" + if len(x) == 1: + # Determine whether x[0] is iterable. + # + # You might want to use collections.abc.Iterable instead of this + # try/except block. Unfortunately, this doesn't work with constexpr. + # + # The problem is that abc.Iterable checks for __iter__ on the *class*. + # But we want constexpr to expose an __iter__ method if and only if the + # wrapped *object* (i.e. self.value) is iterable. Therefore there's no + # right answer for whether the class constexpr defines __iter__, and + # abc.Iterable doesn't work (at least not without some metaclass magic). + try: + iter(x[0]) + return x[0] + except TypeError: + pass + + return x + + +def is_builtin(fn) -> bool: + """Is this a registered triton builtin function?""" + return getattr(fn, TRITON_BUILTIN, False) + + +@builtin +def to_tensor(x, _builder=None): + return semantic.to_tensor(x, _builder) + + +# ----------------------- +# constexpr +# ----------------------- + + +class const: + """ + This class is used as a type annotation to mark pointers to constant data. + The `store` function cannot be called with a pointer to const. Constness + is part of the pointer type and the usual Triton type consistency rules + apply. For example you cannot have a function that returns constant pointer + in one return statement and non-constant pointer in another. + """ + pass + + +class constexpr: + """ + This class is used to store a value that is known at compile-time. + """ + + def __init__(self, value): + if isinstance(value, constexpr): + self.value = value.value + else: + self.value = value + + def __repr__(self) -> str: + return f"constexpr[{self.value}]" + + def __index__(self): + return self.value + + # In interpreter mode, constant values are not wrapped in constexpr, + # and therefore do not have a .value attribute. + # As a result, from here and below, we need to call the _constexpr_to_value + # function to obtain either constexpr.value or the value itself. + def __add__(self, other): + return constexpr(self.value + _constexpr_to_value(other)) + + def __radd__(self, other): + return constexpr(_constexpr_to_value(other) + self.value) + + def __sub__(self, other): + return constexpr(self.value - _constexpr_to_value(other)) + + def __rsub__(self, other): + return constexpr(_constexpr_to_value(other) - self.value) + + def __mul__(self, other): + return constexpr(self.value * _constexpr_to_value(other)) + + def __mod__(self, other): + return constexpr(self.value % _constexpr_to_value(other)) + + def __rmul__(self, other): + return constexpr(_constexpr_to_value(other) * self.value) + + def __truediv__(self, other): + return constexpr(self.value / _constexpr_to_value(other)) + + def __rtruediv__(self, other): + return constexpr(_constexpr_to_value(other) / self.value) + + def __floordiv__(self, other): + return constexpr(self.value // _constexpr_to_value(other)) + + def __rfloordiv__(self, other): + return constexpr(_constexpr_to_value(other) // self.value) + + def __gt__(self, other): + return constexpr(self.value > _constexpr_to_value(other)) + + def __rgt__(self, other): + return constexpr(_constexpr_to_value(other) > self.value) + + def __ge__(self, other): + return constexpr(self.value >= _constexpr_to_value(other)) + + def __rge__(self, other): + return constexpr(_constexpr_to_value(other) >= self.value) + + def __lt__(self, other): + return constexpr(self.value < _constexpr_to_value(other)) + + def __rlt__(self, other): + return constexpr(_constexpr_to_value(other) < self.value) + + def __le__(self, other): + return constexpr(self.value <= _constexpr_to_value(other)) + + def __rle__(self, other): + return constexpr(_constexpr_to_value(other) <= self.value) + + def __eq__(self, other): + return constexpr(self.value == _constexpr_to_value(other)) + + def __ne__(self, other): + return constexpr(self.value != _constexpr_to_value(other)) + + def __bool__(self): + return bool(self.value) + + def __neg__(self): + return constexpr(-self.value) + + def __and__(self, other): + return constexpr(self.value & _constexpr_to_value(other)) + + def logical_and(self, other): + return constexpr(self.value and _constexpr_to_value(other)) + + def __or__(self, other): + return constexpr(self.value | _constexpr_to_value(other)) + + def __xor__(self, other): + return constexpr(self.value ^ _constexpr_to_value(other)) + + def logical_or(self, other): + return constexpr(self.value or _constexpr_to_value(other)) + + def __pos__(self): + return constexpr(+self.value) + + def __invert__(self): + return constexpr(~self.value) + + def __pow__(self, other): + return constexpr(self.value**_constexpr_to_value(other)) + + def __rpow__(self, other): + return constexpr(_constexpr_to_value(other)**self.value) + + def __rshift__(self, other): + return constexpr(self.value >> _constexpr_to_value(other)) + + def __lshift__(self, other): + return constexpr(self.value << _constexpr_to_value(other)) + + def __not__(self): + return constexpr(not self.value) + + def __iter__(self): + return iter(self.value) + + def __call__(self, *args, **kwds): + return self.value(*args, **kwds) + + +CONSTEXPR_0 = constexpr(0) + + +def _unwrap_if_constexpr(o): + return o.value if isinstance(o, constexpr) else o + + +def check_bit_width(value, shift_value): + if isinstance(value, tensor) and isinstance(shift_value, constexpr): + bitwidth = value.type.scalar.primitive_bitwidth + if shift_value.value >= bitwidth: + warn( + f"Value {shift_value.value} exceeds the maximum bitwidth ({bitwidth}) for type '{value.dtype}'. This may result in undefined behavior." + ) + + +# ----------------------- +# dtype +# ----------------------- + + +class dtype: + SINT_TYPES = ['int8', 'int16', 'int32', 'int64'] + UINT_TYPES = ['int1', 'uint8', 'uint16', 'uint32', 'uint64'] + FP_TYPES = ['fp8e4b15', 'fp8e4nv', 'fp8e4b8', 'fp8e5', 'fp8e5b16', 'fp16', 'bf16', 'fp32', 'fp64'] + STANDARD_FP_TYPES = ['fp16', 'bf16', 'fp32', 'fp64'] + OTHER_TYPES = ['void'] + + class SIGNEDNESS(Enum): + SIGNED = 0 + UNSIGNED = 1 + + class KIND(Enum): + BOOLEAN = 0 + INTEGRAL = 1 + FLOATING = 2 + + def __init__(self, name): + name = _unwrap_if_constexpr(name) + self.name = name + assert name in dtype.SINT_TYPES + dtype.UINT_TYPES + dtype.FP_TYPES + dtype.OTHER_TYPES, name + if name in dtype.SINT_TYPES: + self.int_signedness = dtype.SIGNEDNESS.SIGNED + self.int_bitwidth = int(name.split('int')[-1]) + self.primitive_bitwidth = self.int_bitwidth + elif name in dtype.UINT_TYPES: + self.int_signedness = dtype.SIGNEDNESS.UNSIGNED + self.int_bitwidth = int(name.split('int')[-1]) + self.primitive_bitwidth = self.int_bitwidth + elif name in dtype.FP_TYPES: + if name == 'fp8e4b15': + self.fp_mantissa_width = 3 + self.primitive_bitwidth = 8 + self.exponent_bias = 15 + elif name == 'fp8e4nv': + self.fp_mantissa_width = 3 + self.primitive_bitwidth = 8 + self.exponent_bias = 7 + elif name == 'fp8e4b8': + self.fp_mantissa_width = 3 + self.primitive_bitwidth = 8 + self.exponent_bias = 8 + elif name == 'fp8e5': + self.fp_mantissa_width = 2 + self.primitive_bitwidth = 8 + self.exponent_bias = 15 + elif name == 'fp8e5b16': + self.fp_mantissa_width = 2 + self.primitive_bitwidth = 8 + self.exponent_bias = 16 + elif name == 'fp16': + self.fp_mantissa_width = 10 + self.primitive_bitwidth = 16 + self.exponent_bias = 15 + elif name == 'bf16': + self.fp_mantissa_width = 7 + self.primitive_bitwidth = 16 + self.exponent_bias = 127 + elif name == 'fp32': + self.fp_mantissa_width = 23 + self.primitive_bitwidth = 32 + self.exponent_bias = 127 + elif name == 'fp64': + self.fp_mantissa_width = 52 + self.primitive_bitwidth = 64 + self.exponent_bias = 1023 + else: + raise RuntimeError(f'Unsupported floating-point type {name}') + elif name == 'void': + self.primitive_bitwidth = 0 + + def is_fp8(self): + return 'fp8' in self.name + + def is_fp8e4nv(self): + return self.name == 'fp8e4nv' + + def is_fp8e4b8(self): + return self.name == 'fp8e4b8' + + def is_fp8e4b15(self): + return self.name == 'fp8e4b15' + + def is_fp8e5(self): + return self.name == 'fp8e5' + + def is_fp8e5b16(self): + return self.name == 'fp8e5b16' + + def is_fp16(self): + return self.name == 'fp16' + + def is_bf16(self): + return self.name == 'bf16' + + def is_fp32(self): + return self.name == 'fp32' + + def is_fp64(self): + return self.name == 'fp64' + + def is_int1(self): + return self.name == 'int1' + + def is_int8(self): + return self.name == 'int8' + + def is_int16(self): + return self.name == 'int16' + + def is_int32(self): + return self.name == 'int32' + + def is_int64(self): + return self.name == 'int64' + + def is_uint8(self): + return self.name == 'uint8' + + def is_uint16(self): + return self.name == 'uint16' + + def is_uint32(self): + return self.name == 'uint32' + + def is_uint64(self): + return self.name == 'uint64' + + def is_floating(self): + return self.name in dtype.FP_TYPES + + def is_standard_floating(self): + return self.name in dtype.STANDARD_FP_TYPES + + def is_int_signed(self): + return self.name in dtype.SINT_TYPES + + def is_int_unsigned(self): + return self.name in dtype.UINT_TYPES + + def is_int(self): + return self.name in dtype.SINT_TYPES + dtype.UINT_TYPES + + def is_bool(self): + return self.is_int1() + + def kind(self): + # Return int value following the type ordering bool < integer < fp + if self.is_bool(): + return dtype.KIND.BOOLEAN + elif self.is_int(): + return dtype.KIND.INTEGRAL + else: + assert self.is_floating() + return dtype.KIND.FLOATING + + def get_int_max_value(self): + if self.is_int_signed(): + return 2**(self.int_bitwidth - 1) - 1 + if self.is_int_unsigned(): + return 2**self.int_bitwidth - 1 + assert False + + def get_int_min_value(self): + if self.is_int_signed(): + return -2**(self.int_bitwidth - 1) + if self.is_int_unsigned(): + return 0 + assert False + + @staticmethod + def is_dtype(type_str): + return type_str in dtype.SINT_TYPES + dtype.UINT_TYPES + dtype.FP_TYPES + dtype.OTHER_TYPES + + @staticmethod + def is_void(): + raise RuntimeError("Not implemented") + + @staticmethod + def is_block(): + return False + + @staticmethod + def is_ptr(): + return False + + @staticmethod + def is_const(): + return False + + def __eq__(self, other: dtype): + if not isinstance(other, dtype): + return False + return self.name == other.name + + def __ne__(self, other: dtype): + return not self.__eq__(other) + + def __hash__(self): + return hash((self.name, )) + + @property + def scalar(self): + return self + + def to_ir(self, builder: ir.builder) -> ir.type: + if self.name.startswith("fp8"): + if self.name not in builder.options.supported_fp8_dtypes: + raise ValueError(f'type {self} not supported in this architecture. ' + f'The supported fp8 dtypes are {builder.options.supported_fp8_dtypes}') + if self.name in builder.options.deprecated_fp8_dtypes: + warn(f"{self.name} is deprecated in this architecture and will be removed in a future triton release") + + if self.name == 'void': + return builder.get_void_ty() + elif self.name == 'int1': + return builder.get_int1_ty() + elif self.name in ('int8', 'uint8'): + return builder.get_int8_ty() + elif self.name in ('int16', 'uint16'): + return builder.get_int16_ty() + elif self.name in ('int32', 'uint32'): + return builder.get_int32_ty() + elif self.name in ('int64', 'uint64'): + return builder.get_int64_ty() + elif self.name == 'fp8e5': + return builder.get_fp8e5_ty() + elif self.name == 'fp8e5b16': + return builder.get_fp8e5b16_ty() + elif self.name == 'fp8e4nv': + return builder.get_fp8e4nv_ty() + elif self.name == 'fp8e4b8': + return builder.get_fp8e4b8_ty() + elif self.name == 'fp8e4b15': + return builder.get_fp8e4b15_ty() + elif self.name == 'fp16': + return builder.get_half_ty() + elif self.name == 'bf16': + return builder.get_bf16_ty() + elif self.name == 'fp32': + return builder.get_float_ty() + elif self.name == 'fp64': + return builder.get_double_ty() + raise ValueError(f'fail to convert {self} to ir type') + + def __str__(self): + return self.name + + def codegen_name(self): + if self.name.startswith("fp"): + return "float" + self.name[2:] + elif self.name.startswith("bf"): + return "bfloat" + self.name[2:] + else: + return self.name + + @property + def cache_key_part(self) -> str: + """See cache_key_part() in triton.cc.""" + return self.name + + def __repr__(self): + """Output of repr needs to be an evaluatable expression""" + return f'triton.language.{self.codegen_name()}' + + +# Some functions have a param named `dtype`, which shadows the `dtype` class. +# We can't change the param name because it is part of function's public API. +# Declare an alias so those functions can still reference the dtype class. +_DtypeClass = dtype + + +class pointer_type(dtype): + + def __init__(self, element_ty: dtype, address_space: int = 1, const: bool = False): + element_ty = _unwrap_if_constexpr(element_ty) + if not isinstance(element_ty, dtype): + raise TypeError(f'element_ty has type `{type(element_ty).__name__}`; expected `dtype`.') + self.element_ty = element_ty + self.address_space = address_space + self.const = const + self.name = f'pointer<{element_ty}>' if not const else f'const_pointer<{element_ty}>' + + def to_ir(self, builder: ir.builder) -> ir.pointer_type: + return builder.get_ptr_ty(self.element_ty.to_ir(builder), self.address_space) + + def __str__(self): + return self.name + + def __repr__(self): + return self.__str__() + + def is_ptr(self): + return True + + def is_const(self): + return self.const + + def __eq__(self, other: pointer_type) -> bool: + if not isinstance(other, pointer_type): + return False + return self.element_ty == other.element_ty and self.address_space == other.address_space and self.const == other.const + + def __ne__(self, other: pointer_type) -> bool: + return not self.__eq__(other) + + @property + def scalar(self): + return self + + +class nv_tma_desc_type(pointer_type): + + def __init__(self, const=True, address_space=0): + super().__init__(uint8, const=const, address_space=address_space) + self.name = 'nv_tma_desc_type' + + +class block_type(dtype): + + def __init__(self, element_ty: dtype, shape: List): + self.element_ty = element_ty + + # Note that block_type's shape is a list of int + # while tensor's shape is a list of constexpr. + + # shape can be empty ([]) when an input is a 0D tensor. + self.shape = _unwrap_shape(shape) + if not self.shape: + raise TypeError('0d block_type is forbidden') + + self.numel = validate_block_shape(self.shape) + self.name = f'<{self.shape}, {self.element_ty}>' + + def to_ir(self, builder: ir.builder) -> ir.block_type: + return builder.get_block_ty(self.element_ty.to_ir(builder), self.shape) + + def __str__(self): + return self.name + + def __repr__(self): + return self.__str__() + + def is_block(self): + return True + + def get_block_shapes(self) -> List[int]: + return self.shape + + def __eq__(self, other: block_type) -> bool: + if not isinstance(other, block_type): + return False + return self.element_ty == other.element_ty and self.shape == other.shape + + def __ne__(self, other: block_type) -> bool: + return not self.__eq__(other) + + @property + def scalar(self): + return self.element_ty + + +class function_type(dtype): + + def __init__(self, ret_types: List[dtype], param_types: List[dtype]) -> None: + self.ret_types = ret_types + self.param_types = param_types + + def __str__(self): + return f'fn ({self.param_types}) -> {self.ret_types}' + + def to_ir(self, builder: ir.builder): + ir_param_types = [ty.to_ir(builder) for ty in self.param_types] + ret_types = [ret_type.to_ir(builder) for ret_type in self.ret_types] + return builder.get_function_ty(ir_param_types, ret_types) + + +# scalar types +void = dtype('void') +int1 = dtype('int1') +int8 = dtype('int8') +int16 = dtype('int16') +int32 = dtype('int32') +int64 = dtype('int64') +uint8 = dtype('uint8') +uint16 = dtype('uint16') +uint32 = dtype('uint32') +uint64 = dtype('uint64') +float8e5 = dtype('fp8e5') +float8e5b16 = dtype('fp8e5b16') +float8e4nv = dtype('fp8e4nv') +float8e4b8 = dtype('fp8e4b8') +float8e4b15 = dtype('fp8e4b15') +float16 = dtype('fp16') +bfloat16 = dtype('bf16') +float32 = dtype('fp32') +float64 = dtype('fp64') +# pointer types +pi32_t = pointer_type(int32) + + +def get_int_dtype(bitwidth: int, signed: bool) -> dtype: + if bitwidth == 1: + return int1 + elif bitwidth == 8 and signed: + return int8 + elif bitwidth == 8 and not signed: + return uint8 + elif bitwidth == 16 and signed: + return int16 + elif bitwidth == 16 and not signed: + return uint16 + elif bitwidth == 32 and signed: + return int32 + elif bitwidth == 32 and not signed: + return uint32 + elif bitwidth == 64 and signed: + return int64 + elif bitwidth == 64 and not signed: + return uint64 + else: + raise ValueError(f'Unsupported bitwidth {bitwidth} and signedness {signed}') + + +class _value: + """Base class of values that exist in the triton IR (i.e. not constexprs). + """ + + def __init__(self, handle): + self.handle = handle + + +# ----------------------- +# tensor +# ----------------------- + + +class tensor(_value): + """Represents an N-dimensional array of values or pointers. + + :code:`tensor` is the fundamental data structure in Triton programs. Most + functions in :py:mod:`triton.language` operate on and return tensors. + + Most of the named member functions here are duplicates of the free functions + in :code:`triton.language`. For example, :code:`triton.language.sqrt(x)` is + equivalent to :code:`x.sqrt()`. + + :code:`tensor` also defines most of the magic/dunder methods, so you can + write :code:`x+y`, :code:`x << 2`, etc. + + .. rubric:: Constructors + .. + For some reason Sphinx includes __init__ before printing the full table + of methods. Not what I want, but I can't figure out how to fix it. Give + it its own section so it looks intentional. :) + """ + + def __init__(self, handle, type: dtype): + """Not called by user code.""" + # IR handle + super().__init__(handle) + # Block shape + self.shape = type.shape if type.is_block() else () + self.numel = 1 + for s in self.shape: + self.numel *= s + self.numel = constexpr(self.numel) + self.type = type # Tensor type (can be block_type) + # Following the practice in pytorch, dtype is scalar type + self.dtype = type.scalar + self.shape = [constexpr(s) for s in self.shape] + + def __str__(self) -> str: + # ex. "float32[16, 32]" + return str(self.dtype) + '[' + ', '.join(str(s) for s in self.shape) + ']' + + @builtin + def __add__(self, other, _builder=None): + return add(self, other, sanitize_overflow=True, _builder=_builder) + + @builtin + def __radd__(self, other, _builder=None): + return add(other, self, sanitize_overflow=True, _builder=_builder) + + @builtin + def __sub__(self, other, _builder=None): + return sub(self, other, sanitize_overflow=True, _builder=_builder) + + @builtin + def __rsub__(self, other, _builder=None): + return sub(other, self, sanitize_overflow=True, _builder=_builder) + + @builtin + def __mul__(self, other, _builder=None): + return mul(self, other, sanitize_overflow=True, _builder=_builder) + + @builtin + def __rmul__(self, other, _builder=None): + return mul(other, self, sanitize_overflow=True, _builder=_builder) + + @builtin + def __truediv__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.truediv(self, other, _builder) + + @builtin + def __rtruediv__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.truediv(other, self, _builder) + + @builtin + def __floordiv__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.floordiv(self, other, _builder) + + @builtin + def __rfloordiv__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.floordiv(other, self, _builder) + + @builtin + def __mod__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.mod(self, other, _builder) + + @builtin + def __rmod__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.mod(other, self, _builder) + + # unary operators + @builtin + def __neg__(self, _builder=None): + return semantic.minus(self, _builder) + + @builtin + def __invert__(self, _builder=None): + return semantic.invert(self, _builder) + + # bitwise operators + + @builtin + def __and__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.and_(self, other, _builder) + + @builtin + def __rand__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.and_(other, self, _builder) + + @builtin + def __or__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.or_(self, other, _builder) + + @builtin + def __ror__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.or_(other, self, _builder) + + @builtin + def __xor__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.xor_(self, other, _builder) + + @builtin + def __rxor__(self, other, _builder=None): + other = _unwrap_if_constexpr(other) + return semantic.xor_(other, self, _builder) + + @builtin + def __lshift__(self, other, _builder=None): + check_bit_width(self, other) + other = _unwrap_if_constexpr(other) + return semantic.shl(self, other, _builder) + + @builtin + def __rlshift__(self, other, _builder=None): + check_bit_width(other, self) + other = _unwrap_if_constexpr(other) + return semantic.shl(other, self, _builder) + + @builtin + def __rshift__(self, other, _builder=None): + check_bit_width(self, other) + other = _unwrap_if_constexpr(other) + if self.dtype.is_int_signed(): + return semantic.ashr(self, other, _builder) + else: + return semantic.lshr(self, other, _builder) + + @builtin + def __rrshift__(self, other, _builder=None): + check_bit_width(other, self) + other = _unwrap_if_constexpr(other) + if self.dtype.is_int_signed(): + return semantic.ashr(other, self, _builder) + else: + return semantic.lshr(other, self, _builder) + + # > + @builtin + def __gt__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.greater_than(self, other, _builder) + + @builtin + def __rgt__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.greater_than(other, self, _builder) + + # >= + @builtin + def __ge__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.greater_equal(self, other, _builder) + + @builtin + def __rge__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.greater_equal(other, self, _builder) + + # < + @builtin + def __lt__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.less_than(self, other, _builder) + + @builtin + def __rlt__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.less_than(other, self, _builder) + + # <= + @builtin + def __le__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.less_equal(self, other, _builder) + + @builtin + def __rle__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.less_equal(other, self, _builder) + + # == + @builtin + def __eq__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.equal(self, other, _builder) + + @builtin + def __req__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.equal(other, self, _builder) + + @builtin + def __ne__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.not_equal(self, other, _builder) + + @builtin + def __rne__(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.not_equal(other, self, _builder) + + @builtin + def logical_and(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.logical_and(self, other, _builder) + + @builtin + def logical_or(self, other, _builder=None): + other = semantic.to_tensor(other, _builder) + return semantic.logical_or(self, other, _builder) + + # note: __not__ isn't actually a magic method in python + # but it's ok because our ASTVisitor handles it + @builtin + def __not__(self, _builder=None): + return semantic.not_(self, _builder) + + @builtin + def __getitem__(self, slices, _builder=None): + if isinstance(slices, (slice, constexpr)) or slices is None: + slices = [slices] + ret = self + for dim, sl in enumerate(slices): + if sl is None or isinstance(sl, constexpr) and sl.value is None: + ret = semantic.expand_dims(ret, dim, _builder) + elif isinstance(sl, slice) and sl.start is None and sl.stop is None and sl.step is None: + pass + else: + raise ValueError(f"unsupported tensor index: {sl}") + return ret + + @property + def T(self): + """Transposes a 2D tensor.""" + assert False, "Transposition must be created by the AST Visitor" + + @builtin + def to(self, dtype: dtype, fp_downcast_rounding: Optional[str] = None, bitcast: bool = False, _builder=None): + """ + Alias for :py:func:`tensor.cast`. + """ + # Triton doesn't like core functions calling other core functions, so we + # just copy-paste the implementation of cast here. It's not too bad. + dtype = _unwrap_if_constexpr(dtype) + bitcast = _unwrap_if_constexpr(bitcast) + if bitcast: + return semantic.bitcast(self, dtype, _builder) + return semantic.cast(self, dtype, _builder, fp_downcast_rounding) + + # Type stubs for functions added by the _tensor_member_fn decorator. + # (Unfortunately these can't be created automatically.) + # + # We couldn't write these definitions out even if we wanted to, because some + # of these functions are defined in standard.py. + def broadcast_to(self, *shape) -> tensor: + ... + + def trans(self, *dims) -> tensor: + ... + + def permute(self, *dims) -> tensor: + ... + + def split(self) -> tuple[tensor, tensor]: + ... + + def view(self, *shape) -> tensor: + ... + + def reshape(self, *shape) -> tensor: + ... + + def expand_dims(self, axis) -> tensor: + ... + + def cast(self, dtype, fp_downcast_rounding=None, bitcast=False) -> tensor: + ... + + def store(self, value, mask=None, boundary_check=(), cache_modifier="", eviction_policy="") -> tensor: + ... + + def advance(self, offsets) -> tensor: + ... + + def atomic_cas(self, cmp, val, sem=None, scope=None) -> tensor: + ... + + def atomic_xchg(self, val, mask=None, sem=None, scope=None) -> tensor: + ... + + def atomic_add(self, val, mask=None, sem=None, scope=None) -> tensor: + ... + + def atomic_max(self, val, mask=None, sem=None, scope=None) -> tensor: + ... + + def atomic_min(self, val, mask=None, sem=None, scope=None) -> tensor: + ... + + def atomic_and(self, val, mask=None, sem=None, scope=None) -> tensor: + ... + + def atomic_or(self, val, mask=None, sem=None, scope=None) -> tensor: + ... + + def atomic_xor(self, val, mask=None, sem=None, scope=None) -> tensor: + ... + + def exp(self) -> tensor: + ... + + def log(self) -> tensor: + ... + + def cos(self) -> tensor: + ... + + def sin(self) -> tensor: + ... + + def sqrt(self) -> tensor: + ... + + def rsqrt(self) -> tensor: + ... + + def abs(self) -> tensor: + ... + + def reduce(self, axis, combine_fn, keep_dims=False) -> tensor: + ... + + def associative_scan(self, axis, combine_fn, reverse=False) -> tensor: + ... + + def histogram(self, num_bins) -> tensor: + ... + + def cdiv(self, div) -> tensor: + ... + + def sigmoid(self) -> tensor: + ... + + def softmax(self, ieee_rounding=False) -> tensor: + ... + + def ravel(self) -> tensor: + ... + + def max(self, axis=None, return_indices=False, return_indices_tie_break_left=True, keep_dims=False) -> tensor: + ... + + def argmax(self, axis, tie_break_left=True, keep_dims=False) -> tensor: + ... + + def min(self, axis=None, return_indices=False, return_indices_tie_break_left=True, keep_dims=False) -> tensor: + ... + + def argmin(self, axis, tie_break_left=True, keep_dims=False) -> tensor: + ... + + def sum(self, axis=None, keep_dims=False) -> tensor: + ... + + def xor_sum(self, axis=None, keep_dims=False) -> tensor: + ... + + def cumsum(self, axis=0, reverse=False) -> tensor: + ... + + def cumprod(self, axis=0, reverse=False) -> tensor: + ... + + def sort(self, dim: constexpr = None, descending: constexpr = CONSTEXPR_0) -> tensor: + ... + + def flip(self, dim=None) -> tensor: + ... + + +def get_bool_env_var(var_name): + v = os.getenv(var_name, "0") + return v == "1" or v == "true" or v == "on" + + +# ----------------------- +# SPMD Programming Model +# ----------------------- +def _constexpr_to_value(v): + if isinstance(v, constexpr): + return v.value + return v + + +@builtin +def program_id(axis, _builder=None): + """ + Returns the id of the current program instance along the given :code:`axis`. + + :param axis: The axis of the 3D launch grid. Must be 0, 1 or 2. + :type axis: int + """ + # if axis == -1: + # pid0 = program_id(0, _builder) + # pid1 = program_id(1, _builder) + # pid2 = program_id(2, _builder) + # npg0 = num_programs(0, _builder) + # npg1 = num_programs(1, _builder) + # return pid0 + pid1*npg0 + pid2*npg0*npg1 + axis = _constexpr_to_value(axis) + return semantic.program_id(axis, _builder) + + +@builtin +def num_programs(axis, _builder=None): + """ + Returns the number of program instances launched along the given :code:`axis`. + + :param axis: The axis of the 3D launch grid. Must be 0, 1 or 2. + :type axis: int + """ + axis = _constexpr_to_value(axis) + return semantic.num_programs(axis, _builder) + + +# ----------------------- +# Block Initialization +# ----------------------- + + +@builtin +def arange(start, end, _builder=None): + start = _constexpr_to_value(start) + end = _constexpr_to_value(end) + return semantic.arange(start, end, _builder) + + +arange.__doc__ = f""" + Returns contiguous values within the half-open interval :code:`[start, + end)`. :code:`end - start` must be less than or equal to + :code:`TRITON_MAX_TENSOR_NUMEL = {TRITON_MAX_TENSOR_NUMEL}` + + :param start: Start of the interval. Must be a power of two. + :type start: int32 + :param end: End of the interval. Must be a power of two greater than + :code:`start`. + :type end: int32 +""" + + +def _unwrap_shape(shape): + shape = _constexpr_to_value(shape) + return [_constexpr_to_value(s) for s in shape] + + +def _shape_check_impl(shape): + shape = _unwrap_shape(shape) + validate_block_shape(shape) + return shape + + +@builtin +def full(shape, value, dtype, _builder=None): + """ + Returns a tensor filled with the scalar value for the given :code:`shape` and :code:`dtype`. + + :param shape: Shape of the new array, e.g., (8, 16) or (8, ) + :type shape: tuple of ints + :param value: A scalar value to fill the array with + :type value: scalar + :param dtype: Data type of the new array, e.g., :code:`tl.float16` + :type dtype: tl.dtype + """ + shape = _shape_check_impl(shape) + value = _constexpr_to_value(value) + dtype = _constexpr_to_value(dtype) + return semantic.full(shape, value, dtype, _builder) + + +# ----------------------- +# Shape Manipulation +# ----------------------- + + +@builtin +def broadcast(input, other, _builder=None): + """ + Tries to broadcast the two given blocks to a common compatible shape. + + :param input: The first input tensor. + :type input: Block + :param other: The second input tensor. + :type other: Block + """ + return semantic.broadcast_impl_value(input, other, _builder) + + +@_tensor_member_fn +@builtin +def broadcast_to(input, *shape, _builder=None): + """ + Tries to broadcast the given tensor to a new :code:`shape`. + + :param input: The input tensor. + :type input: Block + :param shape: The desired shape. + :type shape: + + :code:`shape` can be passed as a tuple or as individual parameters: :: + + # These are equivalent + broadcast_to(x, (32, 32)) + broadcast_to(x, 32, 32) + """ + shape = _shape_check_impl(_unwrap_iterable(shape)) + return semantic.broadcast_impl_shape(input, shape, _builder) + + +@_tensor_member_fn +@builtin +def trans(input: tensor, *dims, _builder=None): + """ + Permutes the dimensions of a tensor. + + If the parameter :code:`dims` is not specified, the function defaults to a (1,0) permutation, + effectively transposing a 2D tensor. + + :param input: The input tensor. + :param dims: The desired ordering of dimensions. For example, + :code:`(2, 1, 0)` reverses the order dims in a a 3D tensor. + + :code:`dims` can be passed as a tuple or as individual parameters: :: + + # These are equivalent + trans(x, (2, 1, 0)) + trans(x, 2, 1, 0) + + :py:func:`permute` is equivalent to this function, except it doesn't + have the special case when no permutation is specified. + """ + if not dims: + dims = (1, 0) + return semantic.permute(input, dims, _builder) + + +@_tensor_member_fn +@builtin +def permute(input, *dims, _builder=None): + """ + Permutes the dimensions of a tensor. + + :param input: The input tensor. + :type input: Block + :param dims: The desired ordering of dimensions. For example, + :code:`(2, 1, 0)` reverses the order dims in a a 3D tensor. + + :code:`dims` can be passed as a tuple or as individual parameters: :: + + # These are equivalent + permute(x, (2, 1, 0)) + permute(x, 2, 1, 0) + + :py:func:`trans` is equivalent to this function, except when + :code:`dims` is empty, it tries to do a (1,0) permutation. + """ + dims = _unwrap_iterable(dims) + return semantic.permute(input, dims, _builder) + + +@builtin +def cat(input, other, can_reorder=False, _builder=None): + """ + Concatenate the given blocks + + :param input: The first input tensor. + :type input: Tensor + :param other: The second input tensor. + :type other: Tensor + :param reorder: Compiler hint. If true, the compiler is + allowed to reorder elements while concatenating inputs. Only use if the + order does not matter (e.g., result is only used in reduction ops). + Current implementation of `cat` supports only can_reorder=True. + """ + return semantic.cat(input, other, can_reorder, _builder) + + +@builtin +def join(a, b, _builder=None): + """ + Join the given tensors in a new, minor dimension. + + For example, given two tensors of shape (4,8), produces a new tensor of + shape (4,8,2). Given two scalars, returns a tensor of shape (2). + + The two inputs are broadcasted to be the same shape. + + If you want to join more than two elements, you can use multiple calls to + this function. This reflects the constraint in Triton that tensors must + have power-of-two sizes. + + join is the inverse of split. + + :param a: The first input tensor. + :type a: Tensor + :param b: The second input tensor. + :type b: Tensor + """ + return semantic.join(a, b, _builder) + + +@jit +def _take_first(a, b): + return a + + +@_tensor_member_fn +@builtin +def split(a, _builder=None, _generator=None) -> tuple[tensor, tensor]: + """ + Split a tensor in two along its last dim, which must have size 2. + + For example, given a tensor of shape (4,8,2), produces two tensors of shape + (4,8). Given a tensor of shape (2), returns two scalars. + + If you want to split into more than two pieces, you can use multiple calls + to this function (probably plus calling reshape). This reflects the + constraint in Triton that tensors must have power-of-two sizes. + + split is the inverse of join. + + :param a: The tensor to split. + :type a: Tensor + """ + # If len(a.shape) == 1, i.e. a.shape == [2], we should return two scalars. + # But semantic.split can only handle returning tensors. Work around this by + # expanding the input to shape [1,2] and then reducing the result. + was_rank_1 = len(a.shape) == 1 + if was_rank_1: + a = semantic.expand_dims(a, 0, _builder) + + out_lhs, out_rhs = semantic.split(a, _builder) + + if was_rank_1: + # Currently `reduce` is the best way to convert a tensor of shape [1] to a scalar. + out_lhs = typing.cast(tensor, reduce(out_lhs, None, _take_first, _builder=_builder, _generator=_generator)) + out_rhs = typing.cast(tensor, reduce(out_rhs, None, _take_first, _builder=_builder, _generator=_generator)) + + return out_lhs, out_rhs + + +@_tensor_member_fn +@builtin +def view(input, *shape, _builder=None): + """ + Returns a tensor with the same elements as `input` but a different shape. + The order of the elements may not be preserved. + + :param input: The input tensor. + :type input: Block + :param shape: The desired shape. + + :code:`shape` can be passed as a tuple or as individual parameters: :: + + # These are equivalent + view(x, (32, 32)) + view(x, 32, 32) + """ + warn("view is deprecated, please use reshape with can_reorder being true.") + shape = _shape_check_impl(_unwrap_iterable(shape)) + return semantic.reshape(input, shape, can_reorder=True, builder=_builder) + + +@_tensor_member_fn +@builtin +def reshape(input, *shape, can_reorder=False, _builder=None): + """ + Returns a tensor with the same number of elements as input but with the + provided shape. + + :param input: The input tensor. + :type input: Block + :param shape: The new shape. + + :code:`shape` can be passed as a tuple or as individual parameters: :: + + # These are equivalent + reshape(x, (32, 32)) + reshape(x, 32, 32) + """ + shape = _shape_check_impl(_unwrap_iterable(shape)) + return semantic.reshape(input, shape, can_reorder, _builder) + + +def _wrap_axis(axis, ndim): + if not (-ndim <= axis < ndim): + raise ValueError(f"invalid axis {axis}. Expected {-ndim} <= axis < {ndim}") + + return axis if axis >= 0 else axis + ndim + + +@_tensor_member_fn +@builtin +def expand_dims(input, axis, _builder=None): + """ + Expand the shape of a tensor, by inserting new length-1 dimensions. + + Axis indices are with respect to the resulting tensor, so + ``result.shape[axis]`` will be 1 for each axis. + + :param input: The input tensor. + :type input: tl.tensor + :param axis: The indices to add new axes + :type axis: int | Sequence[int] + + """ + input = semantic.to_tensor(input, _builder) + axis = _constexpr_to_value(axis) + axes = list(axis) if isinstance(axis, Sequence) else [axis] + new_ndim = len(input.shape) + len(axes) + axes = [_wrap_axis(_constexpr_to_value(d), new_ndim) for d in axes] + + if len(set(axes)) != len(axes): + raise ValueError(f"expand_dims received duplicate axes, normalized axes = {axes}") + + ret = input + for a in sorted(axes): + ret = semantic.expand_dims(ret, a, _builder) + return ret + + +@_tensor_member_fn +@builtin +def cast(input, dtype: dtype, fp_downcast_rounding: Optional[str] = None, bitcast: bool = False, _builder=None): + """ + Casts a tensor to the given :code:`dtype`. + + :param dtype: The target data type. + :type dtype: tl.dtype + :param fp_downcast_rounding: The rounding mode for downcasting + floating-point values. This parameter is only used when self is a + floating-point tensor and dtype is a floating-point type with a + smaller bitwidth. Supported values are :code:`"rtne"` (round to + nearest, ties to even) and :code:`"rtz"` (round towards zero). + :type fp_downcast_rounding: str, optional + :param bitcast: If true, the tensor is bitcasted to the given + :code:`dtype`, instead of being numerically casted. + :type bitcast: bool, optional + """ + input = semantic.to_tensor(input, _builder) + if isinstance(bitcast, constexpr): + bitcast = bitcast.value + if bitcast: + return semantic.bitcast(input, dtype, _builder) + return semantic.cast(input, dtype, _builder, fp_downcast_rounding) + + +# ----------------------- +# Linear Algebra +# ----------------------- + + +@builtin +def dot(input, other, acc=None, input_precision=None, allow_tf32=None, max_num_imprecise_acc=None, out_dtype=float32, + _builder=None): + """ + Returns the matrix product of two blocks. + + The two blocks must both be two-dimensional or three-dimensional and have compatible inner dimensions. + For three-dimensional blocks, `tl.dot` performs the batched matrix product, + where the first dimension of each block represents the batch dimension. + + :param input: The first tensor to be multiplied. + :type input: 2D or 3D tensor of scalar-type in {:code:`int8`, :code: `float8_e5m2`, :code:`float16`, :code:`bfloat16`, :code:`float32`} + :param other: The second tensor to be multiplied. + :type other: 2D or 3D tensor of scalar-type in {:code:`int8`, :code: `float8_e5m2`, :code:`float16`, :code:`bfloat16`, :code:`float32`} + :param acc: The accumulator tensor. If not None, the result is added to this tensor. + :type acc: 2D or 3D tensor of scalar-type in {:code:`float16`, :code:`float32`, :code:`int32`} + :param input_precision: How to exercise the Tensor Cores for f32 x f32. If + the device does not have Tensor Cores or the inputs are not of dtype f32, + this option is ignored. For devices that do have tensor cores, the + default precision is tf32. + :type input_precision: string. Available options for nvidia: :code:`"tf32"`, :code:`"tf32x3"`, :code:`"ieee"`. Default: :code:`"tf32"`. Avaliable options for amd: :code:`"ieee"`. + :param allow_tf32: *Deprecated.* If true, input_precision is set to "tf32". + Only one of :code:`input_precision` and :code:`allow_tf32` can be + specified (i.e. at least one must be :code:`None`). + """ + assert input_precision is None or allow_tf32 is None, "Only one of input_precision and allow_tf32 can be specified" + if input_precision is None: + supports_tf32 = _builder and "tf32" in _builder.options.allowed_dot_input_precisions + default_precision = "tf32" if (supports_tf32 and (allow_tf32 or allow_tf32 is None)) else "ieee" + input_precision = os.getenv("TRITON_F32_DEFAULT", default_precision) + + input_precision = _constexpr_to_value(input_precision) + out_dtype = _constexpr_to_value(out_dtype) + max_num_imprecise_acc = _constexpr_to_value(max_num_imprecise_acc) + return semantic.dot(input, other, acc, input_precision, max_num_imprecise_acc, out_dtype, _builder) + + +@builtin +def dot_scaled(lhs, lhs_scale, lhs_format, rhs, rhs_scale, rhs_format, acc=None, out_dtype=float32, _builder=None): + """ + Returns the matrix product of two blocks in microscaling format. + lhs and rhs use microscaling formats described here: + https://www.opencompute.org/documents/ocp-microscaling-formats-mx-v1-0-spec-final-pdf + :param lhs: The first tensor to be multiplied. + :type lhs: 2D tensor of f8, f6 or f4 format packed in int32 format. + :param lhs_scale: Scale factor for lhs tensor. + :type lhs_scale: ue8m0 float8 type (currently represented as an int8 tensor). + :param lhs_format: format of the lhs tensor, available formats: {:code:`e4m3`, :code: `e5m2`, :code:`e2m3`, :code:`e3m2`, :code:`e2m1`}. + :param rhs: The second tensor to be multiplied. + :type rhs: 2D tensor of f8, f6 or f4 format packed in int32 format. + :param rhs_scale: Scale factor for rhs tensor. + :type rhs_scale: ue8m0 float8 type (currently represented as an int8 tensor). + :param rhs_format: format of the rhs tensor, available formats: {:code:`e4m3`, :code: `e5m2`, :code:`e2m3`, :code:`e3m2`, :code:`e2m1`}. + :param acc: The accumulator tensor. If not None, the result is added to this tensor. + """ + out_dtype = _constexpr_to_value(out_dtype) + assert out_dtype == float32, "Only float32 is supported for out_dtype at the moment" + return semantic.dot_scaled(lhs, lhs_scale, lhs_format, rhs, rhs_scale, rhs_format, acc, out_dtype, _builder) + + +# ----------------------- +# Non-Atomic Memory Operations +# ----------------------- + + +@builtin +def load(pointer, mask=None, other=None, boundary_check=(), padding_option="", cache_modifier="", eviction_policy="", + volatile=False, _builder=None): + """ + Return a tensor of data whose values are loaded from memory at location defined by `pointer`: + + (1) If `pointer` is a single element pointer, a scalar is be loaded. In + this case: + + - `mask` and `other` must also be scalars, + - `other` is implicitly typecast to `pointer.dtype.element_ty`, and + - `boundary_check` and `padding_option` must be empty. + + (2) If `pointer` is an N-dimensional tensor of pointers, an + N-dimensional tensor is loaded. In this case: + + - `mask` and `other` are implicitly broadcast to `pointer.shape`, + - `other` is implicitly typecast to `pointer.dtype.element_ty`, and + - `boundary_check` and `padding_option` must be empty. + + (3) If `pointer` is a block pointer defined by `make_block_ptr`, a + tensor is loaded. In this case: + + - `mask` and `other` must be `None`, and + - `boundary_check` and `padding_option` can be specified to control the behavior of out-of-bound access. + + :param pointer: Pointer to the data to be loaded + :type pointer: `triton.PointerType`, or block of `dtype=triton.PointerType` + :param mask: if `mask[idx]` is false, do not load the data at address `pointer[idx]` + (must be `None` with block pointers) + :type mask: Block of `triton.int1`, optional + :param other: if `mask[idx]` is false, return `other[idx]` + :type other: Block, optional + :param boundary_check: tuple of integers, indicating the dimensions which should do the boundary check + :type boundary_check: tuple of ints, optional + :param padding_option: should be one of {"", "zero", "nan"}, the padding value to use while out of bounds. "" means an undefined value. + :param cache_modifier: changes cache option in NVIDIA PTX + :type cache_modifier: str, optional, should be one of {"", "ca", "cg"}, where "ca" stands for + cache at all levels and "cg" stands for cache at global level (cache in L2 and below, not L1), see + `cache operator `_ for more details. + :param eviction_policy: changes eviction policy in NVIDIA PTX + :type eviction_policy: str, optional + :param volatile: changes volatile option in NVIDIA PTX + :type volatile: bool, optional + """ + # `mask` and `other` can be constexpr + mask = _constexpr_to_value(mask) + other = _constexpr_to_value(other) + if mask is not None: + mask = semantic.to_tensor(mask, _builder) + if other is not None: + other = semantic.to_tensor(other, _builder) + padding_option = _constexpr_to_value(padding_option) + cache_modifier = _constexpr_to_value(cache_modifier) + eviction_policy = _constexpr_to_value(eviction_policy) + volatile = _constexpr_to_value(volatile) + return semantic.load(pointer, mask, other, boundary_check, padding_option, cache_modifier, eviction_policy, + volatile, _builder) + + +@builtin +def _experimental_descriptor_load(desc_pointer, offsets, shape, dtype, _builder=None): + """ + Experimental feature to access TMA descriptors loads. This is an escape hatch to easily exercise TTGIR operations. + This will be removed in the future and shouldn't be used in production code. + + This loads a tensor of data based on the descriptor and offsets. + """ + type = block_type(_constexpr_to_value(dtype), shape) + return semantic.descriptor_load(desc_pointer, offsets, "", "", type, _builder) + + +@builtin +def _experimental_descriptor_store(desc_pointer, value, offsets, _builder=None): + """ + Experimental feature to access TMA descriptors stores. This is an escape hatch to easily exercise TTGIR operations. + This will be removed in the future and shouldn't be used in production code. + + This stores a tensor of data based on the descriptor and offsets. + """ + return semantic.descriptor_store(desc_pointer, value, offsets, _builder) + + +@_tensor_member_fn +@builtin +def store(pointer, value, mask=None, boundary_check=(), cache_modifier="", eviction_policy="", _builder=None): + """ + Store a tensor of data into memory locations defined by `pointer`. + + (1) If `pointer` is a single element pointer, a scalar is stored. In + this case: + + - `mask` must also be scalar, and + - `boundary_check` and `padding_option` must be empty. + + (2) If `pointer` is an N-dimensional tensor of pointers, an + N-dimensional block is stored. In this case: + + - `mask` is implicitly broadcast to `pointer.shape`, and + - `boundary_check` must be empty. + + (3) If `pointer` is a block pointer defined by `make_block_ptr`, a block + of data is stored. In this case: + + - `mask` must be None, and + - `boundary_check` can be specified to control the behavior of out-of-bound access. + + `value` is implicitly broadcast to `pointer.shape` and typecast to `pointer.dtype.element_ty`. + + :param pointer: The memory location where the elements of `value` are stored + :type pointer: `triton.PointerType`, or block of `dtype=triton.PointerType` + :param value: The tensor of elements to be stored + :type value: Block + :param mask: If `mask[idx]` is false, do not store `value[idx]` at `pointer[idx]` + :type mask: Block of triton.int1, optional + :param boundary_check: tuple of integers, indicating the dimensions which should do the boundary check + :type boundary_check: tuple of ints, optional + :param cache_modifier: changes cache option in NVIDIA PTX + :type cache_modifier: str, optional, should be one of {"", ".wb", ".cg", ".cs", ".wt"}, where ".wb" stands for + cache write-back all coherent levels, ".cg" stands for cache global, ".cs" stands for cache streaming, ".wt" + stands for cache write-through, see `cache operator `_ for more details. + :param eviction_policy: changes eviction policy in NVIDIA PTX + :type eviction_policy: str, optional, should be one of {"", "evict_first", "evict_last"} + """ + # `value` can be constexpr + value = semantic.to_tensor(value, _builder) + mask = _constexpr_to_value(mask) + if mask is not None: + mask = semantic.to_tensor(mask, _builder) + cache_modifier = _constexpr_to_value(cache_modifier) + eviction_policy = _constexpr_to_value(eviction_policy) + return semantic.store(pointer, value, mask, boundary_check, cache_modifier, eviction_policy, _builder) + + +@builtin +def make_block_ptr(base: tensor, shape, strides, offsets, block_shape, order, _builder=None): + """ + Returns a pointer to a block in a parent tensor + + :param base: The base pointer to the parent tensor + :param shape: The shape of the parent tensor + :param strides: The strides of the parent tensor + :param offsets: The offsets to the block + :param block_shape: The shape of the block + :param order: The order of the original data format + """ + return semantic.make_block_ptr(base, shape, strides, offsets, block_shape, order, _builder) + + +@_tensor_member_fn +@builtin +def advance(base, offsets, _builder=None): + """ + Advance a block pointer + + :param base: the block pointer to advance + :param offsets: the offsets to advance, a tuple by dimension + """ + return semantic.advance(base, offsets, _builder) + + +# ----------------------- +# Atomic Memory Operations +# ----------------------- + + +def _add_atomic_docstr(name: str, has_cmp: bool = False) -> Callable[[T], T]: + + def _decorator(func: T) -> T: + docstr = f""" + Performs an atomic {name} at the memory location specified by :code:`pointer`. + + Return the data stored at :code:`pointer` before the atomic operation. + + :param pointer: The memory locations to operate on + :type pointer: Block of dtype=triton.PointerDType""" + if has_cmp: + docstr += """ + :param cmp: The values expected to be found in the atomic object + :type cmp: Block of dtype=pointer.dtype.element_ty""" + docstr += """ + :param val: The values with which to perform the atomic operation + :type val: Block of dtype=pointer.dtype.element_ty + :param sem: Specifies the memory semantics for the operation. Acceptable values are "acquire", + "release", "acq_rel" (stands for "ACQUIRE_RELEASE"), and "relaxed". If not provided, + the function defaults to using "acq_rel" semantics. + :type sem: str, optional + :param scope: Defines the scope of threads that observe the synchronizing effect of the atomic operation. + Acceptable values are "gpu" (default), "cta" (cooperative thread array, thread block), or "sys" (stands for "SYSTEM"). The default value is "gpu". + :type scope: str, optional + """ + func.__doc__ = docstr + return func + + return _decorator + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("compare-and-swap", has_cmp=True) +def atomic_cas(pointer, cmp, val, sem=None, scope=None, _builder=None): + cmp = semantic.to_tensor(cmp, _builder) + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + return semantic.atomic_cas(pointer, cmp, val, sem, scope, _builder) + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("exchange") +def atomic_xchg(pointer, val, mask=None, sem=None, scope=None, _builder=None): + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + mask = _constexpr_to_value(mask) + return semantic.atomic_xchg(pointer, val, mask, sem, scope, _builder) + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("add") +def atomic_add(pointer, val, mask=None, sem=None, scope=None, _builder=None): + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + mask = _constexpr_to_value(mask) + return semantic.atomic_add(pointer, val, mask, sem, scope, _builder) + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("max") +def atomic_max(pointer, val, mask=None, sem=None, scope=None, _builder=None): + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + mask = _constexpr_to_value(mask) + return semantic.atomic_max(pointer, val, mask, sem, scope, _builder) + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("min") +def atomic_min(pointer, val, mask=None, sem=None, scope=None, _builder=None): + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + mask = _constexpr_to_value(mask) + return semantic.atomic_min(pointer, val, mask, sem, scope, _builder) + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("logical and") +def atomic_and(pointer, val, mask=None, sem=None, scope=None, _builder=None): + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + mask = _constexpr_to_value(mask) + return semantic.atomic_and(pointer, val, mask, sem, scope, _builder) + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("logical or") +def atomic_or(pointer, val, mask=None, sem=None, scope=None, _builder=None): + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + mask = _constexpr_to_value(mask) + return semantic.atomic_or(pointer, val, mask, sem, scope, _builder) + + +@_tensor_member_fn +@builtin +@_add_atomic_docstr("logical xor") +def atomic_xor(pointer, val, mask=None, sem=None, scope=None, _builder=None): + val = semantic.to_tensor(val, _builder) + sem = _constexpr_to_value(sem) + scope = _constexpr_to_value(scope) + mask = _constexpr_to_value(mask) + return semantic.atomic_xor(pointer, val, mask, sem, scope, _builder) + + +# ----------------------- +# Conditioning +# ----------------------- + + +@builtin +def where(condition, x, y, _builder=None): + """ + Returns a tensor of elements from either :code:`x` or :code:`y`, depending on :code:`condition`. + + Note that :code:`x` and :code:`y` are always evaluated regardless of the value of :code:`condition`. + + If you want to avoid unintended memory operations, use the :code:`mask` arguments in `triton.load` and `triton.store` instead. + + The shape of :code:`x` and :code:`y` are both broadcast to the shape of :code:`condition`. + :code:`x` and :code:`y` must have the same data type. + + :param condition: When True (nonzero), yield x, otherwise yield y. + :type condition: Block of triton.bool + :param x: values selected at indices where condition is True. + :param y: values selected at indices where condition is False. + """ + condition = semantic.to_tensor(condition, _builder) + x = _unwrap_if_constexpr(x) + y = _unwrap_if_constexpr(y) + return semantic.where(condition, x, y, _builder) + + +# ----------------------- +# Math +# ----------------------- + + +@builtin +def add(x, y, sanitize_overflow: constexpr = True, _builder=None): + x = _unwrap_if_constexpr(x) + y = _unwrap_if_constexpr(y) + return semantic.add(x, y, sanitize_overflow, _builder) + + +@builtin +def sub(x, y, sanitize_overflow: constexpr = True, _builder=None): + x = _unwrap_if_constexpr(x) + y = _unwrap_if_constexpr(y) + return semantic.sub(x, y, sanitize_overflow, _builder) + + +@builtin +def mul(x, y, sanitize_overflow: constexpr = True, _builder=None): + x = _unwrap_if_constexpr(x) + y = _unwrap_if_constexpr(y) + return semantic.mul(x, y, sanitize_overflow, _builder) + + +@builtin +def minimum(x, y, propagate_nan: constexpr = PropagateNan.NONE, _builder=None): + """ + Computes the element-wise minimum of :code:`x` and :code:`y`. + + :param x: the first input tensor + :type x: Block + :param y: the second input tensor + :type y: Block + :param propagate_nan: whether to propagate NaN values. + :type propagate_nan: tl.PropagateNan + + .. seealso:: :class:`tl.PropagateNan` + """ + x = semantic.to_tensor(x, _builder) + y = semantic.to_tensor(y, _builder) + x = _promote_bfloat16_to_float32(x, _builder=_builder) + y = _promote_bfloat16_to_float32(y, _builder=_builder) + propagate_nan = _constexpr_to_value(propagate_nan) + return semantic.minimum(x, y, propagate_nan, _builder) + + +@builtin +def maximum(x, y, propagate_nan: constexpr = PropagateNan.NONE, _builder=None): + """ + Computes the element-wise maximum of :code:`x` and :code:`y`. + + :param x: the first input tensor + :type x: Block + :param y: the second input tensor + :type y: Block + :param propagate_nan: whether to propagate NaN values. + :type propagate_nan: tl.PropagateNan + + .. seealso:: :class:`tl.PropagateNan` + """ + x = semantic.to_tensor(x, _builder) + y = semantic.to_tensor(y, _builder) + x = _promote_bfloat16_to_float32(x, _builder=_builder) + y = _promote_bfloat16_to_float32(y, _builder=_builder) + propagate_nan = _constexpr_to_value(propagate_nan) + return semantic.maximum(x, y, propagate_nan, _builder) + + +@builtin +def clamp(x, min, max, propagate_nan: constexpr = PropagateNan.NONE, _builder=None): + """ + Clamps the input tensor :code:`x` within the range [min, max]. + Behavior when :code:`min` > :code:`max` is undefined. + + :param x: the input tensor + :type x: Block + :param min: the lower bound for clamping + :type min: Block + :param max: the upper bound for clamping + :type max: Block + :param propagate_nan: whether to propagate NaN values. Applies only to the :code:`x` tensor. + If either :code:`min` or :code:`max` is NaN, the result is undefined. + :type propagate_nan: tl.PropagateNan + + .. seealso:: :class:`tl.PropagateNan` + """ + x = semantic.to_tensor(x, _builder) + min = semantic.to_tensor(min, _builder) + max = semantic.to_tensor(max, _builder) + x = _promote_bfloat16_to_float32(x, _builder=_builder) + min = _promote_bfloat16_to_float32(min, _builder=_builder) + max = _promote_bfloat16_to_float32(max, _builder=_builder) + + propagate_nan = _constexpr_to_value(propagate_nan) + + return semantic.clamp(x, min, max, propagate_nan, _builder) + + +# ----------------------- +# Reductions +# ----------------------- + + +def _add_reduction_docstr(name: str, return_indices_arg: str = None, tie_break_arg: str = None) -> Callable[[T], T]: + + def _decorator(func: T) -> T: + docstr = """ + Returns the {name} of all elements in the :code:`input` tensor along the provided :code:`axis` + + :param input: the input values + :type input: Tensor + :param axis: the dimension along which the reduction should be done. If None, reduce all dimensions + :type axis: int + :param keep_dims: if true, keep the reduced dimensions with length 1 + :type keep_dims: bool""" + if return_indices_arg is not None: + docstr += f""" + :param {return_indices_arg}: if true, return index corresponding to the {name} value + :type {return_indices_arg}: bool""" + if tie_break_arg is not None: + docstr += f""" + :param {tie_break_arg}: if true, in case of a tie (i.e., multiple elements have the same {name} value), return the left-most index for values that aren't NaN + :type {tie_break_arg}: bool""" + + func.__doc__ = docstr.format(name=name) + return func + + return _decorator + + +@contextmanager +def _insertion_guard(builder): + ip = builder.get_insertion_point() + yield + builder.restore_insertion_point(ip) + + +@_tensor_member_fn +@builtin +def reduce(input, axis, combine_fn, keep_dims=False, _builder=None, _generator=None): + """Applies the combine_fn to all elements in :code:`input` tensors along the provided :code:`axis` + + :param input: the input tensor, or tuple of tensors + :type input: Tensor + :param axis: the dimension along which the reduction should be done. If None, reduce all dimensions + :type axis: int | None + :param combine_fn: a function to combine two groups of scalar tensors (must be marked with @triton.jit) + :type combine_fn: Callable + :param keep_dims: if true, keep the reduced dimensions with length 1 + :type keep_dims: bool + + """ + if isinstance(input, tensor): + return reduce((input, ), axis, combine_fn, keep_dims=keep_dims, _builder=_builder, _generator=_generator)[0] + + def make_combine_region(reduce_op): + in_scalar_tys = [t.type.scalar for t in input] + prototype = function_type(in_scalar_tys, in_scalar_tys * 2) + + region = reduce_op.get_region(0) + with _insertion_guard(_builder): + param_types = [ty.to_ir(_builder) for ty in prototype.param_types] + block = _builder.create_block_with_parent(region, param_types) + args = [tensor(block.arg(i), ty) for i, ty in enumerate(prototype.param_types)] + results = _generator.call_JitFunction(combine_fn, args, kwargs={}) + if isinstance(results, tensor): + handles = [results.handle] + else: + handles = [r.handle for r in results] + _builder.create_reduce_ret(*handles) + + def expand_ndims(t, ndims): + for _ in builtins.range(ndims): + t = expand_dims(t, 0, _builder=_builder) + return t + + axis = _constexpr_to_value(axis) + keep_dims = _constexpr_to_value(keep_dims) + if axis is not None: + axis = _wrap_axis(axis, len(input[0].shape)) + ret = semantic.reduction(input, axis, make_combine_region, _builder) + if keep_dims: + if axis is not None: + ret = tuple(expand_dims(t, axis, _builder=_builder) for t in ret) + else: + ret = tuple(expand_ndims(t, len(input[0].shape)) for t in ret) + return ret + + +@builtin +def _promote_bfloat16_to_float32(t, _builder=None): + scalar_ty = t.type.scalar + + # hardware doesn't support FMAX, FMIN, CMP for bfloat16 + if scalar_ty is bfloat16: + return t.to(float32, _builder=_builder) + return t + + +@builtin +def _reduce_with_indices(input, axis, combine_fn, keep_dims=False, _builder=None, _generator=None): + axis = _constexpr_to_value(axis) + n = input.shape[axis] + index = arange(0, n, _builder=_builder) + + if len(input.shape) > 1: + # Broadcast index across the non-reduced axes + axes_to_expand = [constexpr(d) for d in builtins.range(len(input.shape))] + del axes_to_expand[axis] + index = expand_dims(index, axes_to_expand, _builder=_builder) + index = broadcast_to(index, input.shape, _builder=_builder) + + rvalue, rindices = reduce((input, index), axis, combine_fn, keep_dims=keep_dims, _builder=_builder, + _generator=_generator) + return rvalue, rindices + + +# ----------------------- +# Scans +# ----------------------- + + +def _add_scan_docstr(name: str) -> Callable[[T], T]: + + def _decorator(func: T) -> T: + docstr = """ + Returns the {name} of all elements in the :code:`input` tensor along the provided :code:`axis` + + :param input: the input values + :type input: Tensor + :param axis: the dimension along which the scan should be done + :type axis: int""" + func.__doc__ = docstr.format(name=name) + return func + + return _decorator + + +@_tensor_member_fn +@builtin +def associative_scan(input, axis, combine_fn, reverse=False, _builder=None, _generator=None): + """Applies the combine_fn to each elements with a carry in :code:`input` tensors along the provided :code:`axis` and update the carry + + :param input: the input tensor, or tuple of tensors + :type input: Tensor + :param axis: the dimension along which the reduction should be done + :type axis: int + :param combine_fn: a function to combine two groups of scalar tensors (must be marked with @triton.jit) + :type combine_fn: Callable + :param reverse: whether to apply the associative scan in the reverse direction along axis + :type reverse: bool + + """ + if isinstance(input, tensor): + return associative_scan((input, ), axis, combine_fn, reverse, _builder=_builder, _generator=_generator)[0] + + def make_combine_region(scan_op): + in_scalar_tys = [t.type.scalar for t in input] + prototype = function_type(in_scalar_tys, in_scalar_tys * 2) + + region = scan_op.get_region(0) + with _insertion_guard(_builder): + param_types = [ty.to_ir(_builder) for ty in prototype.param_types] + block = _builder.create_block_with_parent(region, param_types) + args = [tensor(block.arg(i), ty) for i, ty in enumerate(prototype.param_types)] + results = _generator.call_JitFunction(combine_fn, args, kwargs={}) + if isinstance(results, tensor): + handles = [results.handle] + else: + handles = [r.handle for r in results] + _builder.create_scan_ret(*handles) + + axis = _constexpr_to_value(axis) + if axis is not None: + axis = _wrap_axis(axis, len(input[0].shape)) + return semantic.associative_scan(input, axis, make_combine_region, reverse, _builder) + + +@_tensor_member_fn +@builtin +def histogram(input, num_bins, _builder=None, _generator=None): + """computes an histogram based on input tensor with num_bins bins, the bins have a width of 1 and start at 0. + + :param input: the input tensor + :type input: Tensor + :param num_bins: number of histogram bins + :type num_bins: int + + """ + num_bins = _constexpr_to_value(num_bins) + return semantic.histogram(input, num_bins, _builder) + + +# ----------------------- +# Compiler Hint Ops +# ----------------------- + + +@builtin +def debug_barrier(_builder=None): + ''' + Insert a barrier to synchronize all threads in a block. + ''' + return semantic.debug_barrier(_builder) + + +@builtin +def multiple_of(input, values, _builder=None): + """ + Let the compiler know that the values in :code:`input` are all multiples of :code:`value`. + """ + if isinstance(values, constexpr): + values = [values] + for i, d in enumerate(values): + if not isinstance(d, constexpr): + raise TypeError(f"values element {i} must have type `constexpr`") + if not isinstance(d.value, int): + raise TypeError(f"values element {i} must have type `constexpr[int]`, got `constexpr[{type(d.value)}]") + values = [x.value for x in values] + return semantic.multiple_of(input, values) + + +@builtin +def max_contiguous(input, values, _builder=None): + """ + Let the compiler know that the `value` first values in :code:`input` are contiguous. + """ + if isinstance(values, constexpr): + values = [values] + for i, d in enumerate(values): + if not isinstance(d, constexpr): + raise TypeError(f"values element {i} must have type `constexpr`") + if not isinstance(d.value, int): + raise TypeError(f"values element {i} must have type `constexpr[int]`, got `constexpr[{type(d.value)}]") + values = [x.value for x in values] + return semantic.max_contiguous(input, values) + + +@builtin +def max_constancy(input, values, _builder=None): + """ + Let the compiler know that the `value` first values in :code:`input` are constant. + + e.g. if :code:`values` is [4], then each group of 4 values in :code:`input` should all be equal, + for example [0, 0, 0, 0, 1, 1, 1, 1]. + """ + if isinstance(values, constexpr): + values = [values] + for i, d in enumerate(values): + if not isinstance(d, constexpr): + raise TypeError(f"values element {i} must have type `constexpr`") + if not isinstance(d.value, int): + raise TypeError(f"values element {i} must have type `constexpr[int]`, got `constexpr[{type(d.value)}]") + values = [x.value for x in values] + return semantic.max_constancy(input, values) + + +@builtin +def assume(cond, _builder=None): + ''' + Allow compiler to assume the :code:`cond` is True. + ''' + return semantic.assume(semantic.to_tensor(cond, _builder), _builder) + + +# ----------------------- +# Debugging functions +# ----------------------- + + +@builtin +def static_print(*values, sep: str = " ", end: str = "\n", file=None, flush=False, _builder=None): + ''' + Print the values at compile time. The parameters are the same as the builtin :code:`print`. + + NOTE: Calling the Python builtin :code:`print` is not the same as calling this, it instead maps to :code:`device_print`, + which has special requirements for the arguments. + + .. highlight:: python + .. code-block:: python + + tl.static_print(f"BLOCK_SIZE={BLOCK_SIZE}") + ''' + pass + + +@builtin +def static_assert(cond, msg="", _builder=None): + ''' + Assert the condition at compile time. Does not require that the :code:`TRITON_DEBUG` environment variable + is set. + + .. highlight:: python + .. code-block:: python + + tl.static_assert(BLOCK_SIZE == 1024) + ''' + pass + + +@builtin +def device_print(prefix, *args, hex=False, _builder=None): + ''' + Print the values at runtime from the device. String formatting does not work for runtime values, so you should + provide the values you want to print as arguments. The first value must be a string, all following values must + be scalars or tensors. + + Calling the Python builtin :code:`print` is the same as calling this function, and the requirements for the arguments will match + this function (not the normal requirements for :code:`print`). + + .. highlight:: python + .. code-block:: python + + tl.device_print("pid", pid) + print("pid", pid) + + On CUDA, printfs are streamed through a buffer of limited size (on one host, + we measured the default as 6912 KiB, but this may not be consistent across + GPUs and CUDA versions). If you notice some printfs are being dropped, you + can increase the buffer size by calling + + .. highlight:: python + .. code-block:: python + + triton.runtime.driver.active.utils.set_printf_fifo_size(size_bytes) + + CUDA may raise an error if you try to change this value after running a + kernel that uses printfs. The value set here may only affect the current + device (so if you have multiple GPUs, you'd need to call it multiple times). + + :param prefix: a prefix to print before the values. This is required to be a string literal. + :param args: the values to print. They can be any tensor or scalar. + :param hex: print all values as hex instead of decimal + ''' + import string + prefix = _constexpr_to_value(prefix) + assert isinstance(prefix, str), f"{prefix} is not string" + b_ascii = True + for ch in prefix: + if ch not in string.printable: + b_ascii = False + break + assert b_ascii, f"{prefix} is not an ascii string" + new_args = [] + for arg in args: + new_args.append(semantic.to_tensor(arg, _builder)) + return semantic.device_print(prefix, new_args, hex, _builder) + + +@builtin +def device_assert(cond, msg="", _builder=None): + ''' + Assert the condition at runtime from the device. Requires that the environment variable :code:`TRITON_DEBUG` + is set to a value besides :code:`0` in order for this to have any effect. + + Using the Python :code:`assert` statement is the same as calling this function, except that the second argument + must be provided and must be a string, e.g. :code:`assert pid == 0, "pid != 0"`. The environment variable must + be set for this :code:`assert` statement to have any effect. + + .. highlight:: python + .. code-block:: python + + tl.device_assert(pid == 0) + assert pid == 0, f"pid != 0" + + :param cond: the condition to assert. This is required to be a boolean tensor. + :param msg: the message to print if the assertion fails. This is required to be a string literal. + ''' + msg = _constexpr_to_value(msg) + return semantic.device_assert(semantic.to_tensor(cond, _builder), msg, _builder) + + +@builtin +def inline_asm_elementwise(asm: str, constraints: str, args: Sequence, dtype: Union[dtype, Sequence[dtype]], + is_pure: bool, pack: int, _builder=None): + ''' + Execute inline assembly over a tensor. Essentially, this is :code:`map` + where the function is inline assembly. + + The input tensors :code:`args` are implicitly broadcasted to the same shape. + + :code:`dtype` can be a tuple of types, in which case the output is a + tuple of tensors. + + Each invocation of the inline asm processes :code:`pack` elements at a + time. Exactly which set of inputs a block receives is unspecified. + Input elements of size less than 4 bytes are packed into 4-byte + registers. + + This op does not support empty :code:`dtype` -- the inline asm must + return at least one tensor, even if you don't need it. You can work + around this by returning a dummy tensor of arbitrary type; it shouldn't + cost you anything if you don't use it. + + Example using + `PTX `_ + assembly: + + .. highlight:: python + .. code-block:: python + + @triton.jit + def kernel(A, B, C, D, BLOCK: tl.constexpr): + a = tl.load(A + tl.arange(0, BLOCK)) # uint8 tensor + b = tl.load(B + tl.arange(0, BLOCK)) # float32 tensor + + # For each (a,b) in zip(a,b), perform the following: + # - Let ai be `a` converted to int32. + # - Let af be `a` converted to float. + # - Let m be the max of ai and b. + # - Return ai and mi. + # Do the above 4 elements at a time. + (c, d) = tl.inline_asm_elementwise( + asm=""" + { + // Unpack `a` into `ai`. + .reg .b8 tmp<4>; + mov.b32 {tmp0, tmp1, tmp2, tmp3}, $8; + cvt.u32.u8 $0, tmp0; + cvt.u32.u8 $1, tmp1; + cvt.u32.u8 $2, tmp2; + cvt.u32.u8 $3, tmp3; + } + // Convert `ai` to float. + cvt.rn.f32.s32 $4, $0; + cvt.rn.f32.s32 $5, $1; + cvt.rn.f32.s32 $6, $2; + cvt.rn.f32.s32 $7, $3; + // Take max of `ai` and `b`. + max.f32 $4, $4, $9; + max.f32 $5, $5, $10; + max.f32 $6, $6, $11; + max.f32 $7, $7, $12; + """, + constraints=( + # 8 output registers, namely + # $0=ai0, $1=ai1, $2=ai2, $3=ai3, + # $4=m0, $5=m1, $6=m2, $7=m3. + "=r,=r,=r,=r,=r,=r,=r,=r," + # 5 input registers, namely + # $8=ai, + # $9=b0, $10=b1, $11=b2, $12=b3. + # The four elements from `a` are all packed into one register. + "r,r,r,r,r"), + args=[a, b], + dtype=(tl.int32, tl.float32), + is_pure=True, + pack=4, + ) + tl.store(C + tl.arange(0, BLOCK), c) + tl.store(D + tl.arange(0, BLOCK), d) + + :param asm: assembly to run. Must match target's assembly format. + :param constraints: asm constraints in + `LLVM format `_ + :param args: the input tensors, whose values are passed to the asm block + :param dtype: the element type(s) of the returned tensor(s) + :param is_pure: if true, the compiler assumes the asm block has no side-effects + :param pack: the number of elements to be processed by one instance of inline assembly + :param _builder: the builder + :return: one tensor or a tuple of tensors of the given dtypes + ''' + asm = _constexpr_to_value(asm) + constraints = _constexpr_to_value(constraints) + pack = _constexpr_to_value(pack) + is_pure = _constexpr_to_value(is_pure) + + # Wrap `dtype` in a tuple if it's not already. + try: + iter(dtype) # type: ignore + has_multiple_outputs = True + except TypeError: + has_multiple_outputs = False + dtype = (dtype, ) # type: ignore + + dtype = typing.cast(Sequence[_DtypeClass], dtype) + + res_tys = dtype + if dispatch_args := [semantic.to_tensor(arg, _builder) for arg in args]: + bin_op_type_checking = partial( + semantic.binary_op_type_checking_impl, + builder=_builder, + arithmetic_check=False, + allow_lhs_ptr=True, + allow_rhs_ptr=True, + ) + broadcast_arg = dispatch_args[0] + # Get the broadcast shape over all the arguments + for item in dispatch_args: + _, broadcast_arg = bin_op_type_checking(item, broadcast_arg) + if broadcast_arg.shape: + # Change the shape of each argument based on the broadcast shape + for i, item in enumerate(dispatch_args): + dispatch_args[i], _ = bin_op_type_checking(item, broadcast_arg) + res_tys = [block_type(dt, broadcast_arg.shape) for dt in dtype] + handles = [t.handle for t in dispatch_args] + call = _builder.create_inline_asm(asm, constraints, handles, [ty.to_ir(_builder) for ty in res_tys], is_pure, pack) + + if not has_multiple_outputs: + return tensor(call.get_result(0), res_tys[0]) + return tuple(tensor(call.get_result(i), ty) for i, ty in enumerate(res_tys)) + + +# ----------------------- +# Iterators +# ----------------------- + + +class static_range: + """ + Iterator that counts upward forever. + + .. highlight:: python + .. code-block:: python + + @triton.jit + def kernel(...): + for i in tl.static_range(10): + ... + :note: This is a special iterator used to implement similar semantics to Python's :code:`range` in the context of + :code:`triton.jit` functions. In addition, it also guides the compiler to unroll the loop aggressively. + :param arg1: the start value. + :param arg2: the end value. + :param step: the step value. + """ + + def __init__(self, arg1, arg2=None, step=None): + assert isinstance(arg1, constexpr), f"{arg1} used as tl.static_range start value is not a constexpr" + if step is None: + self.step = constexpr(1) + else: + assert isinstance(step, constexpr), f"{step} used as tl.static_range step value is not a constexpr" + self.step = step + if arg2 is None: + self.start = constexpr(0) + self.end = arg1 + else: + assert isinstance(arg2, constexpr), f"{arg2} used as tl.static_range end value is not a constexpr" + self.start = arg1 + self.end = arg2 + + def __iter__(self): + raise RuntimeError("static_range can only be used in @triton.jit'd functions") + + def __next__(self): + raise RuntimeError("static_range can only be used in @triton.jit'd functions") + + +class range: + """ + Iterator that counts upward forever. + + .. highlight:: python + .. code-block:: python + + @triton.jit + def kernel(...): + for i in tl.range(10, num_stages=3): + ... + :note: This is a special iterator used to implement similar semantics to Python's :code:`range` in the context of + :code:`triton.jit` functions. In addition, it allows user to pass extra attributes to the compiler. + :param arg1: the start value. + :param arg2: the end value. + :param step: the step value. + :param num_stages: pipeline the loop into this many stages (so there are + :code:`num_stages` iterations of the loop in flight at once). + + Note this is subtly different than passing :code:`num_stages` as a + kernel argument. The kernel argument only pipelines loads that feed + into :code:`dot` operations, while this attribute tries to pipeline most + (though not all) loads in this loop. + :param loop_unroll_factor: Tells the Triton IR level loop unroller how many + times to unroll a for loop that this range is used with. Less than 2 for + this value implies no unrolling. + """ + + def __init__(self, arg1, arg2=None, step=None, num_stages=None, loop_unroll_factor=None): + if step is None: + self.step = constexpr(1) + else: + self.step = step + if arg2 is None: + self.start = constexpr(0) + self.end = arg1 + else: + self.start = arg1 + self.end = arg2 + self.num_stages = num_stages + self.loop_unroll_factor = loop_unroll_factor + + def __iter__(self): + raise RuntimeError("tl.range can only be used in @triton.jit'd functions") + + def __next__(self): + raise RuntimeError("tl.range can only be used in @triton.jit'd functions") + + +# ----------------------- +# Extern functions +# ----------------------- + + +def dispatch(func, lib_name: str, lib_path: str, args: list, arg_type_symbol_dict: dict, ret_shape: tuple, + is_pure: bool, _builder=None): + ''' + Dispatch a function to a library + :param func: the function to dispatch + :param lib_name: the name of the library + :param lib_path: the path of the library + :param args: the arguments of the function + :param arg_type_symbol_dict: the type of the arguments + :param ret_shape: the shape of the return value + :param _builder: the builder + :return: the return value of the function + ''' + if len(arg_type_symbol_dict) == 0: + raise ValueError("arg_type_symbol_dict is empty") + + num_args = len(list(arg_type_symbol_dict.keys())[0]) + if len(args) != num_args: + raise ValueError(f"length of input args does not match." + f"Expect {len(args)}, got {num_args}") + + arg_types = [] + arg_list = [] + for arg in args: + if isinstance(arg, tensor): + arg_types.append(arg.dtype) + arg_list.append(arg.handle) + else: + arg_types.append(type(arg)) + arg_list.append(arg) + arg_types = tuple(arg_types) + + if arg_types not in arg_type_symbol_dict: + raise ValueError(f"input arg type does not match." + f"Expect one of {arg_type_symbol_dict.keys()}, got {arg_types}") + else: + symbol = arg_type_symbol_dict[arg_types][0] + ret_type = arg_type_symbol_dict[arg_types][1] + if ret_shape: + ret_type = block_type(ret_type, ret_shape) + return tensor(func(lib_name, lib_path, symbol, arg_list, ret_type.to_ir(_builder), is_pure), ret_type) + + +@builtin +def extern_elementwise(lib_name: str, lib_path: str, args: list, arg_type_symbol_dict: dict, is_pure: bool, + _builder=None): + ''' + Dispatch an elementwise function to a library + :param lib_name: the name of the library + :param lib_path: the path of the library + :param args: the arguments of the function + :param arg_type_symbol_dict: the type of the arguments + :param is_pure: whether the function is pure + :param _builder: the builder + :return: the return value of the function + ''' + dispatch_args = args.copy() + all_scalar = True + ret_shape = None + arg_types = [] + for i in builtins.range(len(dispatch_args)): + dispatch_args[i] = semantic.to_tensor(dispatch_args[i], _builder) + arg_types.append(dispatch_args[i].dtype) + if dispatch_args[i].type.is_block(): + all_scalar = False + if len(arg_types) > 0: + arg_types = tuple(arg_types) + arithmetic_check = True + # If there's a type tuple that is not supported by the library, we will do arithmetic check + if arg_types in arg_type_symbol_dict: + arithmetic_check = False + broadcast_arg = dispatch_args[0] + # Get the broadcast shape over all the arguments + for item in dispatch_args: + _, broadcast_arg = semantic.binary_op_type_checking_impl(item, broadcast_arg, _builder, + arithmetic_check=arithmetic_check) + # Change the shape of each argument based on the broadcast shape + for i in builtins.range(len(dispatch_args)): + dispatch_args[i], _ = semantic.binary_op_type_checking_impl(dispatch_args[i], broadcast_arg, _builder, + arithmetic_check=arithmetic_check) + if not all_scalar: + ret_shape = broadcast_arg.shape + func = _builder.create_extern_elementwise + return dispatch(func, lib_name, lib_path, dispatch_args, arg_type_symbol_dict, ret_shape, is_pure, _builder) + + +def binary_op_type_legalization(lhs, rhs, builder): + ''' + Convert both operands to a single common type + :param lhs: the left operand + :param rhs: the right operand + :param builder: the builder + ''' + return semantic.binary_op_type_checking_impl(lhs, rhs, builder) + + +def extern(fn): + """A decorator for external functions.""" + return builtin(fn) diff --git a/lib/python3.12/site-packages/triton/language/extra/__init__.py b/lib/python3.12/site-packages/triton/language/extra/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3f8c70a716a3da3473a4906b44aec7d35fcc35a5 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/__init__.py @@ -0,0 +1,26 @@ +import pkgutil +from importlib.util import module_from_spec +from sys import modules + +_backends = [] +for module_finder, module_name, is_pkg in pkgutil.iter_modules( + __path__, + prefix=__name__ + ".", +): + # skip .py files (like libdevice.py) + if not is_pkg: + continue + + # import backends (like cuda and hip) that are included during setup.py + spec = module_finder.find_spec(module_name) + if spec is None or spec.loader is None: + continue + module = module_from_spec(spec) + spec.loader.exec_module(module) + + _backends.append(module_name) + modules[module_name] = module + +__all__ = _backends + +del _backends diff --git a/lib/python3.12/site-packages/triton/language/extra/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ddf21c0028dbb9ebad01b8d030ca2a31d81b4db4 Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/__pycache__/libdevice.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/__pycache__/libdevice.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f9298e359fddfa8723683f8a941b3e376f64eef6 Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/__pycache__/libdevice.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/__init__.py b/lib/python3.12/site-packages/triton/language/extra/cuda/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..9fffa216b06de8578f868e240ec134e8213956b7 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/cuda/__init__.py @@ -0,0 +1,13 @@ +from . import libdevice + +from .utils import (globaltimer, num_threads, num_warps, smid, convert_custom_float8_sm70, convert_custom_float8_sm80) + +from ._experimental_tma import * # noqa: F403 +from ._experimental_tma import __all__ as _tma_all + +__all__ = [ + "libdevice", "globaltimer", "num_threads", "num_warps", "smid", "convert_custom_float8_sm70", + "convert_custom_float8_sm80", *_tma_all +] + +del _tma_all diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b353652514deef96a7b3e5a5b3e8fc34c6cb478e Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/_experimental_tma.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/_experimental_tma.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8b0fca7d4992c14d7266b9f92b055d5ac265eac7 Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/_experimental_tma.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/libdevice.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/libdevice.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..389e26de3eb93fee6ad4ca0ea542cd249f984c9c Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/libdevice.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/utils.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/utils.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3ce05b83224a8c6cd2136cca3eaed9eb048ec54c Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/cuda/__pycache__/utils.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/_experimental_tma.py b/lib/python3.12/site-packages/triton/language/extra/cuda/_experimental_tma.py new file mode 100644 index 0000000000000000000000000000000000000000..5677810194d9021a0e3f1490a73b8008b97ab72f --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/cuda/_experimental_tma.py @@ -0,0 +1,108 @@ +from typing import Sequence + +from triton.language import core +from triton.language import semantic +from triton._C.libtriton import ir + +__all__ = [ + "experimental_device_tensormap_create1d", + "experimental_device_tensormap_create2d", + "experimental_tensormap_fenceproxy_acquire", +] + + +def _determine_elem_type(element_ty: core.dtype): + if element_ty.primitive_bitwidth == 8: + return 0 + elif element_ty.primitive_bitwidth == 16: + return 1 + elif element_ty.primitive_bitwidth == 32: + return 2 + else: + raise ValueError("element_ty must be a primitive of size 1, 2, or 4 bytes but got") + + +@core.builtin +def experimental_device_tensormap_create1d( + desc_ptr: core.tensor, + global_address: core.tensor, + load_size: core.tensor, + global_size: core.tensor, + element_ty: core.dtype, + _builder: ir.builder, +): + load_size = core._constexpr_to_value(load_size) + global_size = semantic.to_tensor(global_size, _builder) + element_ty = core._constexpr_to_value(element_ty) + element_stride = [core.full([], 1, core.int32, _builder=_builder)] + + semantic.tensormap_create( + desc_ptr=desc_ptr, + global_address=global_address, + box_dim=[semantic.to_tensor(load_size, _builder)], + global_dim=[global_size], + global_stride=[], + element_stride=element_stride, + elem_type=_determine_elem_type(element_ty), + interleave_layout=0, + swizzle_mode=0, + fill_mode=0, + builder=_builder, + ) + + +@core.builtin +def experimental_device_tensormap_create2d( + desc_ptr: core.tensor, + global_address: core.tensor, + load_size: Sequence[core.constexpr], + global_size: Sequence[core.tensor], + element_ty: core.dtype, + _builder: ir.builder, +): + assert len(load_size) == 2 + assert len(global_size) == 2 + load_size = [core._constexpr_to_value(x) for x in load_size] + global_size = [semantic.to_tensor(x, _builder) for x in global_size] + + element_size = element_ty.primitive_bitwidth // 8 + element_size_t = core.full([], element_size, core.int64, _builder=_builder) + global_stride = semantic.mul(element_size_t, global_size[-1], True, _builder) + # Undocumented, but global_stride seems to be divided by 16 + global_stride = semantic.ashr(global_stride, semantic.to_tensor(4, _builder), _builder) + + contig_dim_size_in_bytes = element_size * load_size[-1] + if contig_dim_size_in_bytes > 128: + load_size[-1] = 128 // element_size + + elem_stride = core.full([], 1, core.int32, _builder=_builder) + + semantic.tensormap_create( + desc_ptr=desc_ptr, + global_address=global_address, + box_dim=[semantic.to_tensor(x, _builder) for x in load_size[::-1]], + global_dim=global_size[::-1], + global_stride=[global_stride], + element_stride=[elem_stride, elem_stride], + elem_type=_determine_elem_type(element_ty), + interleave_layout=0, + swizzle_mode=_determine_swizzle_mode_2d(contig_dim_size_in_bytes, load_size), + fill_mode=0, + builder=_builder, + ) + + +def _determine_swizzle_mode_2d(contig_dim_size_in_bytes, load_size): + if contig_dim_size_in_bytes >= 128: + return 3 + elif contig_dim_size_in_bytes >= 64: + return 2 + elif contig_dim_size_in_bytes >= 32: + return 1 + else: + raise ValueError("block size too small") + + +@core.builtin +def experimental_tensormap_fenceproxy_acquire(desc_ptr: core.tensor, _builder: ir.builder): + semantic.tensormap_fenceproxy_acquire(desc_ptr, _builder) diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/libdevice.py b/lib/python3.12/site-packages/triton/language/extra/cuda/libdevice.py new file mode 100644 index 0000000000000000000000000000000000000000..37e810bb18347728d74d9ab222e07c05e1ce4cd9 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/cuda/libdevice.py @@ -0,0 +1,1629 @@ +from triton.language import core + + +@core.extern +def clz(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("int32"), ): ("__nv_clz", core.dtype("int32")), + (core.dtype("int64"), ): ("__nv_clzll", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def popc(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("int32"), ): ("__nv_popc", core.dtype("int32")), + (core.dtype("int64"), ): ("__nv_popcll", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def byte_perm(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise("", "", [arg0, arg1, arg2], { + (core.dtype("int32"), core.dtype("int32"), core.dtype("int32")): ("__nv_byte_perm", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def mulhi(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("int32"), core.dtype("int32")): ("__nv_mulhi", core.dtype("int32")), + (core.dtype("uint32"), core.dtype("uint32")): ("__nv_umulhi", core.dtype("uint32")), + (core.dtype("int64"), core.dtype("int64")): ("__nv_mul64hi", core.dtype("int64")), + (core.dtype("uint64"), core.dtype("uint64")): ("__nv_umul64hi", core.dtype("uint64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def mul24(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("int32"), core.dtype("int32")): ("__nv_mul24", core.dtype("int32")), + (core.dtype("uint32"), core.dtype("uint32")): ("__nv_umul24", core.dtype("uint32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def brev(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("int32"), ): ("__nv_brev", core.dtype("int32")), + (core.dtype("int64"), ): ("__nv_brevll", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sad(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("int32"), core.dtype("int32"), core.dtype("uint32")): ("__nv_sad", core.dtype("int32")), + (core.dtype("uint32"), core.dtype("uint32"), core.dtype("uint32")): ("__nv_usad", core.dtype("uint32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def abs(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("int32"), ): ("__nv_abs", core.dtype("int32")), + (core.dtype("int64"), ): ("__nv_llabs", core.dtype("int64")), + (core.dtype("fp32"), ): ("__nv_fabsf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_fabs", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def floor(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_floorf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_floor", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rcp64h(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_rcp64h", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rsqrt(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_rsqrtf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_rsqrt", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ceil(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_ceil", core.dtype("fp64")), + (core.dtype("fp32"), ): ("__nv_ceilf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def trunc(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_trunc", core.dtype("fp64")), + (core.dtype("fp32"), ): ("__nv_truncf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def exp2(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_exp2f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_exp2", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def saturatef(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_saturatef", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fma_rn(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmaf_rn", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__nv_fma_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fma_rz(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmaf_rz", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__nv_fma_rz", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fma_rd(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmaf_rd", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__nv_fma_rd", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fma_ru(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmaf_ru", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__nv_fma_ru", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_dividef(arg0, arg1, _builder=None): + return core.extern_elementwise("", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fast_fdividef", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def div_rn(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fdiv_rn", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_ddiv_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def div_rz(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fdiv_rz", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_ddiv_rz", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def div_rd(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fdiv_rd", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_ddiv_rd", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def div_ru(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fdiv_ru", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_ddiv_ru", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rcp_rn(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_frcp_rn", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_drcp_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rcp_rz(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_frcp_rz", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_drcp_rz", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rcp_rd(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_frcp_rd", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_drcp_rd", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rcp_ru(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_frcp_ru", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_drcp_ru", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sqrt_rn(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fsqrt_rn", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_dsqrt_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sqrt_rz(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fsqrt_rz", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_dsqrt_rz", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sqrt_rd(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fsqrt_rd", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_dsqrt_rd", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sqrt_ru(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fsqrt_ru", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_dsqrt_ru", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sqrt(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_sqrtf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_sqrt", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def add_rn(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dadd_rn", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fadd_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def add_rz(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dadd_rz", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fadd_rz", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def add_rd(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dadd_rd", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fadd_rd", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def add_ru(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dadd_ru", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fadd_ru", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def mul_rn(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dmul_rn", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmul_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def mul_rz(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dmul_rz", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmul_rz", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def mul_rd(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dmul_rd", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmul_rd", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def mul_ru(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + arg1, + ], { + ( + core.dtype("fp64"), + core.dtype("fp64"), + ): ("__nv_dmul_ru", core.dtype("fp64")), + ( + core.dtype("fp32"), + core.dtype("fp32"), + ): ("__nv_fmul_ru", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2float_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2float_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2float_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2float_rz", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2float_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2float_rd", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2float_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2float_ru", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2int_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2int_rn", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2int_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2int_rz", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2int_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2int_rd", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2int_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2int_ru", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2uint_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2uint_rn", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2uint_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2uint_rz", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2uint_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2uint_rd", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2uint_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2uint_ru", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def int2double_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int32"), ): ("__nv_int2double_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def uint2double_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint32"), ): ("__nv_uint2double_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2int_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2int_rn", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2int_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2int_rz", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2int_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2int_rd", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2int_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2int_ru", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2uint_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2uint_rn", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2uint_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2uint_rz", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2uint_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2uint_rd", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2uint_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2uint_ru", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def int2float_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int32"), ): ("__nv_int2float_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def int2float_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int32"), ): ("__nv_int2float_rz", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def int2float_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int32"), ): ("__nv_int2float_rd", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def int2float_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int32"), ): ("__nv_int2float_ru", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def uint2float_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint32"), ): ("__nv_uint2float_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def uint2float_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint32"), ): ("__nv_uint2float_rz", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def uint2float_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint32"), ): ("__nv_uint2float_rd", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def uint2float_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint32"), ): ("__nv_uint2float_ru", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def hiloint2double(arg0, arg1, _builder=None): + return core.extern_elementwise("", "", [arg0, arg1], { + (core.dtype("int32"), core.dtype("int32")): ("__nv_hiloint2double", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2loint(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2loint", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2hiint(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2hiint", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ll_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ll_rn", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ll_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ll_rz", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ll_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ll_rd", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ll_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ll_ru", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ull_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ull_rn", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ull_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ull_rz", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ull_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ull_rd", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float2ull_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float2ull_ru", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ll_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ll_rn", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ll_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ll_rz", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ll_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ll_rd", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ll_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ll_ru", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ull_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ull_rn", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ull_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ull_rz", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ull_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ull_rd", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double2ull_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double2ull_ru", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2float_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2float_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2float_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2float_rz", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2float_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2float_rd", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2float_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2float_ru", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2float_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2float_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2float_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2float_rz", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2float_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2float_rd", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2float_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2float_ru", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2double_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2double_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2double_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2double_rz", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2double_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2double_rd", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ll2double_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_ll2double_ru", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2double_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2double_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2double_rz(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2double_rz", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2double_rd(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2double_rd", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ull2double_ru(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint64"), ): ("__nv_ull2double_ru", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def int_as_float(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int32"), ): ("__nv_int_as_float", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float_as_int(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float_as_int", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def uint_as_float(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("uint32"), ): ("__nv_uint_as_float", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def float_as_uint(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_float_as_uint", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def longlong_as_double(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("int64"), ): ("__nv_longlong_as_double", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def double_as_longlong(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_double_as_longlong", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_sinf(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_sinf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_cosf(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_cosf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_log2f(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_log2f", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_logf(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_logf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_expf(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_expf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_tanf(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_tanf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_exp10f(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_exp10f", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_log10f(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_fast_log10f", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_powf(arg0, arg1, _builder=None): + return core.extern_elementwise("", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fast_powf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def hadd(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("int32"), core.dtype("int32")): ("__nv_hadd", core.dtype("int32")), + (core.dtype("uint32"), core.dtype("uint32")): ("__nv_uhadd", core.dtype("uint32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rhadd(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("int32"), core.dtype("int32")): ("__nv_rhadd", core.dtype("int32")), + (core.dtype("uint32"), core.dtype("uint32")): ("__nv_urhadd", core.dtype("uint32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sub_rn(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fsub_rn", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dsub_rn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sub_rz(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fsub_rz", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dsub_rz", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sub_rd(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fsub_rd", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dsub_rd", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sub_ru(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fsub_ru", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_dsub_ru", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rsqrt_rn(arg0, _builder=None): + return core.extern_elementwise("", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__nv_frsqrt_rn", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ffs(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("int32"), ): ("__nv_ffs", core.dtype("int32")), + (core.dtype("int64"), ): ("__nv_ffsll", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rint(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__nv_rintf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_rint", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def llrint(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__nv_llrintf", core.dtype("int64")), + (core.dtype("fp64"), ): ("__nv_llrint", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def nearbyint(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__nv_nearbyintf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_nearbyint", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def isnan(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__nv_isnanf", core.dtype("int32")), + (core.dtype("fp64"), ): ("__nv_isnand", core.dtype("int32")), + }, is_pure=True, _builder=_builder).to(core.int1, _builder=_builder) + + +@core.extern +def signbit(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__nv_signbitf", core.dtype("int32")), + (core.dtype("fp64"), ): ("__nv_signbitd", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def copysign(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_copysignf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_copysign", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def finitef(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_finitef", core.dtype("int32")), + }, is_pure=True, _builder=_builder).to(core.int1, _builder=_builder) + + +@core.extern +def isinf(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_isinff", core.dtype("int32")), + (core.dtype("fp64"), ): ("__nv_isinfd", core.dtype("int32")), + }, is_pure=True, _builder=_builder).to(core.int1, _builder=_builder) + + +@core.extern +def nextafter(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_nextafterf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_nextafter", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sin(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_sinf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_sin", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cos(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_cosf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_cos", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sinpi(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_sinpif", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_sinpi", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cospi(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_cospif", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_cospi", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def tan(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_tanf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_tan", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log2(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_log2f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_log2", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def exp(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_expf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_exp", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def exp10(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_exp10f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_exp10", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cosh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_coshf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_cosh", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sinh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_sinhf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_sinh", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def tanh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_tanhf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_tanh", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def atan2(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_atan2f", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_atan2", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def atan(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_atanf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_atan", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def asin(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_asinf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_asin", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def acos(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_acosf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_acos", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_logf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_log", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log10(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_log10f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_log10", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log1p(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_log1pf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_log1p", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def acosh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_acoshf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_acosh", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def asinh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_asinhf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_asinh", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def atanh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_atanhf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_atanh", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def expm1(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_expm1f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_expm1", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def hypot(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_hypotf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_hypot", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rhypot(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_rhypotf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_rhypot", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def norm3d(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__nv_norm3df", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__nv_norm3d", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rnorm3d(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__nv_rnorm3df", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__nv_rnorm3d", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def norm4d(arg0, arg1, arg2, arg3, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2, arg3], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): + ("__nv_norm4df", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): + ("__nv_norm4d", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rnorm4d(arg0, arg1, arg2, arg3, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2, arg3], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): + ("__nv_rnorm4df", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): + ("__nv_rnorm4d", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cbrt(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_cbrtf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_cbrt", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rcbrt(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_rcbrtf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_rcbrt", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def j0(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_j0f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_j0", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def j1(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_j1f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_j1", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def y0(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_y0f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_y0", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def y1(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_y1f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_y1", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def yn(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("int32"), core.dtype("fp32")): ("__nv_ynf", core.dtype("fp32")), + (core.dtype("int32"), core.dtype("fp64")): ("__nv_yn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def jn(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("int32"), core.dtype("fp32")): ("__nv_jnf", core.dtype("fp32")), + (core.dtype("int32"), core.dtype("fp64")): ("__nv_jn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cyl_bessel_i0(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_cyl_bessel_i0f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_cyl_bessel_i0", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cyl_bessel_i1(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_cyl_bessel_i1f", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_cyl_bessel_i1", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erf(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_erff", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_erf", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erfinv(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_erfinvf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_erfinv", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erfc(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_erfcf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_erfc", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erfcx(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_erfcxf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_erfcx", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erfcinv(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_erfcinvf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_erfcinv", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def normcdfinv(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_normcdfinvf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_normcdfinv", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def normcdf(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_normcdff", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_normcdf", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def lgamma(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_lgammaf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_lgamma", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ldexp(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("int32")): ("__nv_ldexpf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("int32")): ("__nv_ldexp", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def scalbn(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("int32")): ("__nv_scalbnf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("int32")): ("__nv_scalbn", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fmod(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmodf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_fmod", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def remainder(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_remainderf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_remainder", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fma(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__nv_fmaf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__nv_fma", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def pow(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("int32")): ("__nv_powif", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("int32")): ("__nv_powi", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_powf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_pow", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def tgamma(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_tgammaf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_tgamma", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def round(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_roundf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_round", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def llround(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_llroundf", core.dtype("int64")), + (core.dtype("fp64"), ): ("__nv_llround", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fdim(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__nv_fdimf", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__nv_fdim", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ilogb(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_ilogbf", core.dtype("int32")), + (core.dtype("fp64"), ): ("__nv_ilogb", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def logb(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__nv_logbf", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__nv_logb", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def isfinited(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp64"), ): ("__nv_isfinited", core.dtype("int32")), + }, is_pure=True, _builder=_builder).to(core.int1, _builder=_builder) diff --git a/lib/python3.12/site-packages/triton/language/extra/cuda/utils.py b/lib/python3.12/site-packages/triton/language/extra/cuda/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..01bc040b289feea13190672677571b6e7dffa08f --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/cuda/utils.py @@ -0,0 +1,109 @@ +from triton.language import core + + +@core.extern +def globaltimer(_builder=None): + return core.inline_asm_elementwise("mov.u64 $0, %globaltimer;", "=l", [], dtype=core.int64, is_pure=False, pack=1, + _builder=_builder) + + +@core.extern +def smid(_builder=None): + return core.inline_asm_elementwise("mov.u32 $0, %smid;", "=r", [], dtype=core.int32, is_pure=True, pack=1, + _builder=_builder) + + +@core.builtin +def num_threads(_builder=None): + return core.constexpr(_builder.options.num_warps * 32) + + +@core.builtin +def num_warps(_builder=None): + return core.constexpr(_builder.options.num_warps) + + +# ----- FP8E4M3B15 ------ +# This data-type is a variant of the standard FP8E4M3 format. +# It was designed for fast software conversion to FP16 on +# nvidia GPUs that do not support it natively. +# This is the same format as FP8E4M3Nv, but: +# - the exponent bias is 15 instead of 7 +# - 0xff and 0x7f are mapped to +-1.750 instead of +-nan +@core.builtin +def convert_fp8e4b15_to_float16(arg, _builder=None): + return core.inline_asm_elementwise( + "{ \n" + ".reg .b32 a<2>, b<2>; \n" + "prmt.b32 a0, 0, $2, 0x5746; \n" + "and.b32 b0, a0, 0x7f007f00; \n" + "and.b32 b1, a0, 0x00ff00ff; \n" + "and.b32 a1, a0, 0x00800080; \n" + "shr.b32 b0, b0, 1; \n" + "add.u32 b1, b1, a1; \n" + "lop3.b32 $0, b0, 0x80008000, a0, 0xf8; \n" + "shl.b32 $1, b1, 7; \n" + "} \n", "=r,=r,r", [arg], dtype=core.float16, is_pure=True, pack=4, + _builder=_builder) + + +@core.builtin +def convert_float16_to_fp8e4b15(arg, has_minx2, _builder=None): + asm = """{ + .reg .pred p<4>; + .reg .b32 a<2>, b<2>; + .reg .b16 c<4>; + .reg .b16 max_val_f16; + .reg .b32 max_val_f16x2; + mov.b16 max_val_f16, 0x3F00; + mov.b32 max_val_f16x2, 0x3F003F00; + and.b32 a0, $1, 0x7fff7fff; + and.b32 a1, $2, 0x7fff7fff;""" + if has_minx2: + asm += """min.f16x2 a0, a0, max_val_f16x2; + min.f16x2 a1, a1, max_val_f16x2;""" + else: + asm += """setp.lt.f16x2 p0|p1, a0, max_val_f16x2; + setp.lt.f16x2 p2|p3, a1, max_val_f16x2; + mov.b32 {c0, c1}, a0; + mov.b32 {c2, c3}, a1; + selp.b16 c0, c0, max_val_f16, p0; + selp.b16 c1, c1, max_val_f16, p1; + selp.b16 c2, c2, max_val_f16, p2; + selp.b16 c3, c3, max_val_f16, p3; + mov.b32 a0, {c0, c1}; + mov.b32 a1, {c2, c3};""" + asm += """mad.lo.u32 a0, a0, 2, 0x00800080; + mad.lo.u32 a1, a1, 2, 0x00800080; + lop3.b32 b0, $1, 0x80008000, a0, 0xea; + lop3.b32 b1, $2, 0x80008000, a1, 0xea; + prmt.b32 $0, b0, b1, 0x7531; + }""" + return core.inline_asm_elementwise(asm, "=r,r,r", [arg], dtype=core.float8e4b15, is_pure=True, pack=4, + _builder=_builder) + + +@core.builtin +def convert_custom_float8(arg, dst_ty, fp_downcast_rounding, has_minx2, _builder=None): + if arg.type.scalar.is_fp8e4b15(): + upcast_val = convert_fp8e4b15_to_float16(arg, _builder=_builder) + if dst_ty.scalar.is_fp32(): + upcast_val = upcast_val.to(core.float32, _builder=_builder) + return upcast_val + + assert arg.type.scalar.is_fp16() or arg.type.scalar.is_fp32() + downcast_val = arg + if arg.type.scalar.is_fp32(): + downcast_val = downcast_val.to(core.float16, fp_downcast_rounding="rtz", _builder=_builder) + downcast_val = convert_float16_to_fp8e4b15(downcast_val, has_minx2=has_minx2, _builder=_builder) + return downcast_val + + +@core.builtin +def convert_custom_float8_sm80(arg, dst_ty, fp_downcast_rounding=None, _builder=None): + return convert_custom_float8(arg, dst_ty, fp_downcast_rounding, has_minx2=True, _builder=_builder) + + +@core.builtin +def convert_custom_float8_sm70(arg, dst_ty, fp_downcast_rounding=None, _builder=None): + return convert_custom_float8(arg, dst_ty, fp_downcast_rounding, has_minx2=False, _builder=_builder) diff --git a/lib/python3.12/site-packages/triton/language/extra/hip/__init__.py b/lib/python3.12/site-packages/triton/language/extra/hip/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..229b57d87d6580c73f74dcb1556d33c0a1a2df60 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/hip/__init__.py @@ -0,0 +1,3 @@ +from . import libdevice + +__all__ = ["libdevice"] diff --git a/lib/python3.12/site-packages/triton/language/extra/hip/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/hip/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ebde25d86ee33e045e48a16b32cff9e758ce2ed6 Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/hip/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/hip/__pycache__/libdevice.cpython-312.pyc b/lib/python3.12/site-packages/triton/language/extra/hip/__pycache__/libdevice.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..48b1596cb9fa93bfeca8c13a6ed8ee85fa6ebefe Binary files /dev/null and b/lib/python3.12/site-packages/triton/language/extra/hip/__pycache__/libdevice.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/language/extra/hip/libdevice.py b/lib/python3.12/site-packages/triton/language/extra/hip/libdevice.py new file mode 100644 index 0000000000000000000000000000000000000000..a69d4406cc128786ca53039f4188327664312e65 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/hip/libdevice.py @@ -0,0 +1,475 @@ +from triton.language import core + + +@core.extern +def abs(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("int32"), ): ("__triton_hip_iabs", core.dtype("int32")), + (core.dtype("int64"), ): ("__triton_hip_iabs", core.dtype("int64")), + (core.dtype("fp32"), ): ("__triton_hip_fabs", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__triton_hip_fabs", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def floor(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_floor_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_floor_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def rsqrt(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_rsqrt_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_rsqrt_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ceil(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_ceil_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_ceil_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def trunc(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_trunc_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_trunc_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def exp2(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_exp2_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_exp2_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def exp(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_exp_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_exp_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_expf(arg0, _builder=None): + return core.extern_elementwise("", "", [arg0], { + (core.dtype("fp32"), ): ("__triton_hip_fast_expf", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fast_dividef(arg0, arg1, _builder=None): + return core.extern_elementwise("", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__triton_hip_fast_fdividef", core.dtype("fp32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sqrt(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_sqrt_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_sqrt_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def llrint(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__triton_hip_llrint", core.dtype("int64")), + (core.dtype("fp64"), ): ("__triton_hip_llrint", core.dtype("int64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def nearbyint(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__ocml_nearbyint_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_nearbyint_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def isnan(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__ocml_isnan_f32", core.dtype("int32")), + (core.dtype("fp64"), ): ("__ocml_isnan_f64", core.dtype("int32")), + }, is_pure=True, _builder=_builder).to(core.int1, _builder=_builder) + + +@core.extern +def signbit(arg0, _builder=None): + return core.extern_elementwise( + "", "", [ + arg0, + ], { + (core.dtype("fp32"), ): ("__ocml_signbit_f32", core.dtype("int32")), + (core.dtype("fp64"), ): ("__ocml_signbit_f64", core.dtype("int32")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def copysign(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__ocml_copysign_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__ocml_copysign_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def isinf(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_isinf_f32", core.dtype("int32")), + (core.dtype("fp64"), ): ("__ocml_isinf_f64", core.dtype("int32")), + }, is_pure=True, _builder=_builder).to(core.int1, _builder=_builder) + + +@core.extern +def nextafter(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__ocml_nextafter_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__ocml_nextafter_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sin(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_sin_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_sin_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cos(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_cos_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_cos_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def tan(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_tan_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_tan_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log2(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_log2_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_log2_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cosh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_cosh_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_cosh_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def sinh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_sinh_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_sinh_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def tanh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_tanh_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_tanh_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def atan2(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__ocml_atan2_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__ocml_atan2_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def atan(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_atan_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_atan_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def asin(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_asin_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_asin_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def acos(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_acos_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_acos_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_log_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_log_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log10(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_log10_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_log10_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def log1p(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_log1p_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_log1p_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def acosh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_acosh_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_acosh_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def asinh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_asinh_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_asinh_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def atanh(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_atanh_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_atanh_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def expm1(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_expm1_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_expm1_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def hypot(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__ocml_hypot_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__ocml_hypot_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def j0(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_j0_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_j0_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def j1(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_j1_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_j1_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def y0(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_y0_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_y0_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def y1(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_y1_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_y1_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cyl_bessel_i0(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_i0_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_i0_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def cyl_bessel_i1(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_i1_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_i1_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erf(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_erf_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_erf_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erfinv(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_erfinv_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_erfinv_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erfc(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_erfc_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_erfc_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def erfcx(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_erfcx_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_erfcx_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def lgamma(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_lgamma_f32", core.dtype("fp32")), + (core.dtype("fp64"), ): ("__ocml_lgamma_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ldexp(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("int32")): ("__ocml_ldexp_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("int32")): ("__ocml_ldexp_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fmod(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("fp32")): ("__ocml_fmod_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__ocml_fmod_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def fma(arg0, arg1, arg2, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1, arg2], { + (core.dtype("fp32"), core.dtype("fp32"), core.dtype("fp32")): ("__ocml_fma_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64"), core.dtype("fp64")): ("__ocml_fma_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def pow(arg0, arg1, _builder=None): + return core.extern_elementwise( + "", "", [arg0, arg1], { + (core.dtype("fp32"), core.dtype("int32")): ("__ocml_pown_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("int32")): ("__ocml_pown_f64", core.dtype("fp64")), + (core.dtype("fp32"), core.dtype("fp32")): ("__ocml_pow_f32", core.dtype("fp32")), + (core.dtype("fp64"), core.dtype("fp64")): ("__ocml_pow_f64", core.dtype("fp64")), + }, is_pure=True, _builder=_builder) + + +@core.extern +def ilogb(arg0, _builder=None): + return core.extern_elementwise( + "", "", [arg0], { + (core.dtype("fp32"), ): ("__ocml_ilogb_f32", core.dtype("int32")), + (core.dtype("fp64"), ): ("__ocml_ilogb_f64", core.dtype("int32")), + }, is_pure=True, _builder=_builder) diff --git a/lib/python3.12/site-packages/triton/language/extra/libdevice.py b/lib/python3.12/site-packages/triton/language/extra/libdevice.py new file mode 100644 index 0000000000000000000000000000000000000000..76627035de78dc02d80a5879f1a40a49bf352a89 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/extra/libdevice.py @@ -0,0 +1,786 @@ +def clz(arg0): + ... + + +def popc(arg0): + ... + + +def byte_perm(arg0, arg1, arg2): + ... + + +def mulhi(arg0, arg1): + ... + + +def mul24(arg0, arg1): + ... + + +def brev(arg0): + ... + + +def sad(arg0, arg1, arg2): + ... + + +def abs(arg0): + ... + + +def floor(arg0): + ... + + +def rcp64h(arg0): + ... + + +def rsqrt(arg0): + ... + + +def ceil(arg0): + ... + + +def trunc(arg0): + ... + + +def exp2(arg0): + ... + + +def saturatef(arg0): + ... + + +def fma_rn(arg0, arg1, arg2): + ... + + +def fma_rz(arg0, arg1, arg2): + ... + + +def fma_rd(arg0, arg1, arg2): + ... + + +def fma_ru(arg0, arg1, arg2): + ... + + +def fast_dividef(arg0, arg1): + ... + + +def div_rn(arg0, arg1): + ... + + +def div_rz(arg0, arg1): + ... + + +def div_rd(arg0, arg1): + ... + + +def div_ru(arg0, arg1): + ... + + +def rcp_rn(arg0): + ... + + +def rcp_rz(arg0): + ... + + +def rcp_rd(arg0): + ... + + +def rcp_ru(arg0): + ... + + +def sqrt_rn(arg0): + ... + + +def sqrt_rz(arg0): + ... + + +def sqrt_rd(arg0): + ... + + +def sqrt_ru(arg0): + ... + + +def sqrt(arg0): + ... + + +def add_rn(arg0, arg1): + ... + + +def add_rz(arg0, arg1): + ... + + +def add_rd(arg0, arg1): + ... + + +def add_ru(arg0, arg1): + ... + + +def mul_rn(arg0, arg1): + ... + + +def mul_rz(arg0, arg1): + ... + + +def mul_rd(arg0, arg1): + ... + + +def mul_ru(arg0, arg1): + ... + + +def double2float_rn(arg0): + ... + + +def double2float_rz(arg0): + ... + + +def double2float_rd(arg0): + ... + + +def double2float_ru(arg0): + ... + + +def double2int_rn(arg0): + ... + + +def double2int_rz(arg0): + ... + + +def double2int_rd(arg0): + ... + + +def double2int_ru(arg0): + ... + + +def double2uint_rn(arg0): + ... + + +def double2uint_rz(arg0): + ... + + +def double2uint_rd(arg0): + ... + + +def double2uint_ru(arg0): + ... + + +def int2double_rn(arg0): + ... + + +def uint2double_rn(arg0): + ... + + +def float2int_rn(arg0): + ... + + +def float2int_rz(arg0): + ... + + +def float2int_rd(arg0): + ... + + +def float2int_ru(arg0): + ... + + +def float2uint_rn(arg0): + ... + + +def float2uint_rz(arg0): + ... + + +def float2uint_rd(arg0): + ... + + +def float2uint_ru(arg0): + ... + + +def int2float_rn(arg0): + ... + + +def int2float_rz(arg0): + ... + + +def int2float_rd(arg0): + ... + + +def int2float_ru(arg0): + ... + + +def uint2float_rn(arg0): + ... + + +def uint2float_rz(arg0): + ... + + +def uint2float_rd(arg0): + ... + + +def uint2float_ru(arg0): + ... + + +def hiloint2double(arg0, arg1): + ... + + +def double2loint(arg0): + ... + + +def double2hiint(arg0): + ... + + +def float2ll_rn(arg0): + ... + + +def float2ll_rz(arg0): + ... + + +def float2ll_rd(arg0): + ... + + +def float2ll_ru(arg0): + ... + + +def float2ull_rn(arg0): + ... + + +def float2ull_rz(arg0): + ... + + +def float2ull_rd(arg0): + ... + + +def float2ull_ru(arg0): + ... + + +def double2ll_rn(arg0): + ... + + +def double2ll_rz(arg0): + ... + + +def double2ll_rd(arg0): + ... + + +def double2ll_ru(arg0): + ... + + +def double2ull_rn(arg0): + ... + + +def double2ull_rz(arg0): + ... + + +def double2ull_rd(arg0): + ... + + +def double2ull_ru(arg0): + ... + + +def ll2float_rn(arg0): + ... + + +def ll2float_rz(arg0): + ... + + +def ll2float_rd(arg0): + ... + + +def ll2float_ru(arg0): + ... + + +def ull2float_rn(arg0): + ... + + +def ull2float_rz(arg0): + ... + + +def ull2float_rd(arg0): + ... + + +def ull2float_ru(arg0): + ... + + +def ll2double_rn(arg0): + ... + + +def ll2double_rz(arg0): + ... + + +def ll2double_rd(arg0): + ... + + +def ll2double_ru(arg0): + ... + + +def ull2double_rn(arg0): + ... + + +def ull2double_rz(arg0): + ... + + +def ull2double_rd(arg0): + ... + + +def ull2double_ru(arg0): + ... + + +def int_as_float(arg0): + ... + + +def float_as_int(arg0): + ... + + +def uint_as_float(arg0): + ... + + +def float_as_uint(arg0): + ... + + +def longlong_as_double(arg0): + ... + + +def double_as_longlong(arg0): + ... + + +def fast_sinf(arg0): + ... + + +def fast_cosf(arg0): + ... + + +def fast_log2f(arg0): + ... + + +def fast_logf(arg0): + ... + + +def fast_expf(arg0): + ... + + +def fast_tanf(arg0): + ... + + +def fast_exp10f(arg0): + ... + + +def fast_log10f(arg0): + ... + + +def fast_powf(arg0, arg1): + ... + + +def hadd(arg0, arg1): + ... + + +def rhadd(arg0, arg1): + ... + + +def sub_rn(arg0, arg1): + ... + + +def sub_rz(arg0, arg1): + ... + + +def sub_rd(arg0, arg1): + ... + + +def sub_ru(arg0, arg1): + ... + + +def rsqrt_rn(arg0): + ... + + +def ffs(arg0): + ... + + +def rint(arg0): + ... + + +def llrint(arg0): + ... + + +def nearbyint(arg0): + ... + + +def isnan(arg0): + ... + + +def signbit(arg0): + ... + + +def copysign(arg0, arg1): + ... + + +def finitef(arg0): + ... + + +def isinf(arg0): + ... + + +def nextafter(arg0, arg1): + ... + + +def sin(arg0): + ... + + +def cos(arg0): + ... + + +def sinpi(arg0): + ... + + +def cospi(arg0): + ... + + +def tan(arg0): + ... + + +def log2(arg0): + ... + + +def exp(arg0): + ... + + +def exp10(arg0): + ... + + +def cosh(arg0): + ... + + +def sinh(arg0): + ... + + +def tanh(arg0): + ... + + +def atan2(arg0, arg1): + ... + + +def atan(arg0): + ... + + +def asin(arg0): + ... + + +def acos(arg0): + ... + + +def log(arg0): + ... + + +def log10(arg0): + ... + + +def log1p(arg0): + ... + + +def acosh(arg0): + ... + + +def asinh(arg0): + ... + + +def atanh(arg0): + ... + + +def expm1(arg0): + ... + + +def hypot(arg0, arg1): + ... + + +def rhypot(arg0, arg1): + ... + + +def norm3d(arg0, arg1, arg2): + ... + + +def rnorm3d(arg0, arg1, arg2): + ... + + +def norm4d(arg0, arg1, arg2, arg3): + ... + + +def rnorm4d(arg0, arg1, arg2, arg3): + ... + + +def cbrt(arg0): + ... + + +def rcbrt(arg0): + ... + + +def j0(arg0): + ... + + +def j1(arg0): + ... + + +def y0(arg0): + ... + + +def y1(arg0): + ... + + +def yn(arg0, arg1): + ... + + +def jn(arg0, arg1): + ... + + +def cyl_bessel_i0(arg0): + ... + + +def cyl_bessel_i1(arg0): + ... + + +def erf(arg0): + ... + + +def erfinv(arg0): + ... + + +def erfc(arg0): + ... + + +def erfcx(arg0): + ... + + +def erfcinv(arg0): + ... + + +def normcdfinv(arg0): + ... + + +def normcdf(arg0): + ... + + +def lgamma(arg0): + ... + + +def ldexp(arg0, arg1): + ... + + +def scalbn(arg0, arg1): + ... + + +def fmod(arg0, arg1): + ... + + +def remainder(arg0, arg1): + ... + + +def fma(arg0, arg1, arg2): + ... + + +def pow(arg0, arg1): + ... + + +def tgamma(arg0): + ... + + +def round(arg0): + ... + + +def llround(arg0): + ... + + +def fdim(arg0, arg1): + ... + + +def ilogb(arg0): + ... + + +def logb(arg0): + ... + + +def isfinited(arg0): + ... diff --git a/lib/python3.12/site-packages/triton/language/math.py b/lib/python3.12/site-packages/triton/language/math.py new file mode 100644 index 0000000000000000000000000000000000000000..60ba170c78792e88e4da04327e330700781bc3a1 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/math.py @@ -0,0 +1,250 @@ +from . import core +from . import semantic +from functools import wraps +from typing import List + +T = core.TypeVar('T') + + +def _check_dtype(dtypes: List[str]) -> T: + """ + We're following libdevice's convention to check accepted data types for math functions. + It is not a good practice to support all data types as accelerators/GPUs don't support + many float16 and bfloat16 math operations. + We should let the users know that they are using and invoke explicit cast to convert + the data type to the supported one. + """ + + def wrapper(fn): + + @wraps(fn) + def check(*args, **kwargs): + # concatenate args and kwargs + all_args = list(args) + list(kwargs.values()) + for arg in [a for a in all_args if isinstance(a, core.tensor)]: + if arg.type.scalar.name not in dtypes: + raise ValueError(f"Expected dtype {dtypes} but got {arg.type.scalar.name}") + return fn(*args, **kwargs) + + return check + + return wrapper + + +def _add_math_1arg_docstr(name: str) -> core.Callable[[T], T]: + + def _decorator(func: T) -> T: + docstr = """ + Computes the element-wise {name} of :code:`x`. + + :param x: the input values + :type x: Block + """ + func.__doc__ = docstr.format(name=name) + return func + + return _decorator + + +def _add_math_2arg_docstr(name: str) -> core.Callable[[T], T]: + + def _decorator(func: T) -> T: + docstr = """ + Computes the element-wise {name} of :code:`x` and :code:`y`. + + :param x: the input values + :type x: Block + :param y: the input values + :type y: Block + """ + func.__doc__ = docstr.format(name=name) + return func + + return _decorator + + +def _add_math_3arg_docstr(name: str) -> core.Callable[[T], T]: + + def _decorator(func: T) -> T: + docstr = """ + Computes the element-wise {name} of :code:`x`, :code:`y`, and :code:`z`. + + :param x: the input values + :type x: Block + :param y: the input values + :type y: Block + :param z: the input values + :type z: Block + """ + func.__doc__ = docstr.format(name=name) + return func + + return _decorator + + +@core.builtin +@_check_dtype(dtypes=["int32", "int64", "uint32", "uint64"]) +@_add_math_2arg_docstr("most significant N bits of the 2N-bit product") +def umulhi(x, y, _builder=None): + x = semantic.to_tensor(x, _builder) + y = semantic.to_tensor(y, _builder) + x, y = core.binary_op_type_legalization(x, y, _builder) + return core.tensor(_builder.create_umulhi(x.handle, y.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("exponential") +@core._tensor_member_fn +def exp(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_exp(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("exponential (base 2)") +@core._tensor_member_fn +def exp2(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_exp2(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("natural logarithm") +@core._tensor_member_fn +def log(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_log(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("logarithm (base 2)") +@core._tensor_member_fn +def log2(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_log2(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("cosine") +@core._tensor_member_fn +def cos(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_cos(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("sine") +@core._tensor_member_fn +def sin(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_sin(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("fast square root") +@core._tensor_member_fn +def sqrt(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_sqrt(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32"]) +@_add_math_1arg_docstr("precise square root (rounding to nearest wrt the IEEE standard)") +@core._tensor_member_fn +def sqrt_rn(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_precise_sqrt(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("inverse square root") +@core._tensor_member_fn +def rsqrt(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_rsqrt(x.handle), x.type) + + +@core.builtin +@_add_math_1arg_docstr("absolute value") +@core._tensor_member_fn +def abs(x, _builder=None): + x = semantic.to_tensor(x, _builder) + dtype = x.dtype + if dtype.is_fp8e4b15(): + mask = core.full(x.shape, 0x7F, core.int8, _builder=_builder) + return core.tensor(_builder.create_and(x.handle, mask.handle), x.type) + elif dtype.is_floating(): + return core.tensor(_builder.create_fabs(x.handle), x.type) + elif dtype.is_int_signed(): + return core.tensor(_builder.create_iabs(x.handle), x.type) + elif dtype.is_int_unsigned(): + return x # no-op + else: + assert False, f"Unexpected dtype {dtype}" + + +@core.builtin +@_add_math_2arg_docstr("fast division") +def fdiv(x, y, ieee_rounding=False, _builder=None): + ieee_rounding = core._constexpr_to_value(ieee_rounding) + x = semantic.to_tensor(x, _builder) + y = semantic.to_tensor(y, _builder) + return semantic.fdiv(x, y, ieee_rounding, _builder) + + +@core.builtin +@_check_dtype(dtypes=["fp32"]) +@_add_math_2arg_docstr("precise division (rounding to nearest wrt the IEEE standard)") +def div_rn(x, y, _builder=None): + x = semantic.to_tensor(x, _builder) + y = semantic.to_tensor(y, _builder) + x, y = core.binary_op_type_legalization(x, y, _builder) + return core.tensor(_builder.create_precise_divf(x.handle, y.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("error function") +@core._tensor_member_fn +def erf(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_erf(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("floor") +@core._tensor_member_fn +def floor(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_floor(x.handle), x.type) + + +@core.builtin +@_check_dtype(dtypes=["fp32", "fp64"]) +@_add_math_1arg_docstr("ceil") +@core._tensor_member_fn +def ceil(x, _builder=None): + x = semantic.to_tensor(x, _builder) + return core.tensor(_builder.create_ceil(x.handle), x.type) + + +@core.builtin +@_add_math_3arg_docstr("fused multiply-add") +def fma(x, y, z, _builder=None): + x = semantic.to_tensor(x, _builder) + y = semantic.to_tensor(y, _builder) + z = semantic.to_tensor(z, _builder) + x, y = core.binary_op_type_legalization(x, y, _builder) + z, x = core.binary_op_type_legalization(z, x, _builder) + z, y = core.binary_op_type_legalization(z, y, _builder) + return core.tensor(_builder.create_fma(x.handle, y.handle, z.handle), x.type) diff --git a/lib/python3.12/site-packages/triton/language/random.py b/lib/python3.12/site-packages/triton/language/random.py new file mode 100644 index 0000000000000000000000000000000000000000..1c001695e189341bcd7670137eb975c60a378151 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/random.py @@ -0,0 +1,207 @@ +from ..runtime.jit import jit +from . import core as tl +from . import math + +N_ROUNDS_DEFAULT = 10 # Default number of rounds for philox + +# ------------------- +# randint +# ------------------- + + +@jit +def philox_impl(c0, c1, c2, c3, k0, k1, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + """ + Run `n_rounds` rounds of Philox for state (c0, c1, c2, c3) and key (k0, k1). + """ + if c0.dtype == tl.uint32: + PHILOX_KEY_A: tl.constexpr = 0x9E3779B9 + PHILOX_KEY_B: tl.constexpr = 0xBB67AE85 + PHILOX_ROUND_A: tl.constexpr = 0xD2511F53 + PHILOX_ROUND_B: tl.constexpr = 0xCD9E8D57 + else: + tl.static_assert(c0.dtype == tl.uint64, "dtype not supported in philox_impl") + PHILOX_KEY_A: tl.constexpr = 0x9E3779B97F4A7C15 + PHILOX_KEY_B: tl.constexpr = 0xBB67AE8584CAA73B + PHILOX_ROUND_A: tl.constexpr = 0xD2E7470EE14C6C93 + PHILOX_ROUND_B: tl.constexpr = 0xCA5A826395121157 + + for _ in tl.static_range(n_rounds): + # for _ in range(n_rounds): + # update random state + A = PHILOX_ROUND_A + B = PHILOX_ROUND_B + _c0, _c2 = c0, c2 + c0 = math.umulhi(B, _c2) ^ c1 ^ k0 + c2 = math.umulhi(A, _c0) ^ c3 ^ k1 + c1 = tl.mul(B, _c2, sanitize_overflow=False) + c3 = tl.mul(A, _c0, sanitize_overflow=False) + # raise key + k0 = tl.add(k0, PHILOX_KEY_A, sanitize_overflow=False) + k1 = tl.add(k1, PHILOX_KEY_B, sanitize_overflow=False) + return c0, c1, c2, c3 + + +@jit +def philox(seed, c0, c1, c2, c3, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + seed = tl.to_tensor(seed) + c0 = tl.to_tensor(c0) + c1 = tl.to_tensor(c1) + c2 = tl.to_tensor(c2) + c3 = tl.to_tensor(c3) + seed = seed.to(tl.uint64) + if tl.constexpr(c0.dtype.primitive_bitwidth) == 32: + int_dtype = tl.uint32 + seed_hi = ((seed >> 32) & 0xffffffff).to(tl.uint32) + seed_lo = (seed & 0xffffffff).to(tl.uint32) + else: + tl.static_assert(tl.constexpr(c0.dtype.primitive_bitwidth) == 64, "bitwidth not supported in philox") + int_dtype = tl.uint64 + seed_hi = tl.full((1, ), 0, dtype=int_dtype) + seed_lo = seed + c0 = c0.to(int_dtype, bitcast=True) + c1 = c1.to(int_dtype, bitcast=True) + c2 = c2.to(int_dtype, bitcast=True) + c3 = c3.to(int_dtype, bitcast=True) + return philox_impl(c0, c1, c2, c3, seed_lo, seed_hi, n_rounds) + + +@jit +def randint(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + """ + Given a :code:`seed` scalar and an :code:`offset` block, returns a single + block of random :code:`int32`. + + If you need multiple streams of random numbers, + using `randint4x` is likely to be faster than calling `randint` 4 times. + + :param seed: The seed for generating random numbers. + :param offset: The offsets to generate random numbers for. + """ + ret, _, _, _ = randint4x(seed, offset, n_rounds) + return ret + + +@jit +def randint4x(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + """ + Given a :code:`seed` scalar and an :code:`offset` block, returns four + blocks of random :code:`int32`. + + This is the maximally efficient entry point + to Triton's Philox pseudo-random number generator. + + :param seed: The seed for generating random numbers. + :param offsets: The offsets to generate random numbers for. + """ + # _0 = tl.zeros(offset.shape, offset.dtype) + _0 = offset * 0 + return philox(seed, offset, _0, _0, _0, n_rounds) + + +# ------------------- +# rand +# ------------------- + +# @jit +# def uint32_to_uniform_float(x): +# """ +# Numerically stable function to convert a random uint32 into a random float uniformly sampled in [0, 1). +# """ +# two_to_the_minus_32: tl.constexpr = 2.328306e-10 +# return x * two_to_the_minus_32 + + +@jit +def uint_to_uniform_float(x): + """ + Numerically stable function to convert a random uint into a random float uniformly sampled in [0, 1). + """ + # TODO: fix frontend issues and cleanup + # conditions can be simplified + # scale is ((2**23 - 1) / 2**23) * 2**(N_BITS - 1) + if tl.constexpr(x.dtype == tl.uint32) or tl.constexpr(x.dtype == tl.int32): + # maximum value such that `MAX_INT * scale < 1.0` (with float rounding) + x = x.to(tl.int32, bitcast=True) + scale = 4.6566127342e-10 + else: + tl.static_assert(tl.constexpr(x.dtype == tl.uint64) or tl.constexpr(x.dtype == tl.int64)) + x = x.to(tl.int64, bitcast=True) + scale = 1.0842020432385337e-19 + x = tl.where(x < 0, -x - 1, x) + return x * scale + + +@jit +def rand(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + """ + Given a :code:`seed` scalar and an :code:`offset` block, + returns a block of random :code:`float32` in :math:`U(0, 1)`. + + :param seed: The seed for generating random numbers. + :param offsets: The offsets to generate random numbers for. + """ + source = randint(seed, offset, n_rounds) + return uint_to_uniform_float(source) + + +@jit +def rand4x(seed, offsets, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + """ + Given a :code:`seed` scalar and an :code:`offsets` block, + returns 4 blocks of random :code:`float32` in :math:`U(0, 1)`. + + :param seed: The seed for generating random numbers. + :param offsets: The offsets to generate random numbers for. + """ + i1, i2, i3, i4 = randint4x(seed, offsets, n_rounds) + u1 = uint_to_uniform_float(i1) + u2 = uint_to_uniform_float(i2) + u3 = uint_to_uniform_float(i3) + u4 = uint_to_uniform_float(i4) + return u1, u2, u3, u4 + + +# ------------------- +# randn +# ------------------- + + +@jit +def pair_uniform_to_normal(u1, u2): + """Box-Muller transform""" + u1 = tl.maximum(1.0e-7, u1) + th = 6.283185307179586 * u2 + r = math.sqrt(-2.0 * math.log(u1)) + return r * math.cos(th), r * math.sin(th) + + +@jit +def randn(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + """ + Given a :code:`seed` scalar and an :code:`offset` block, + returns a block of random :code:`float32` in :math:`\\mathcal{N}(0, 1)`. + + :param seed: The seed for generating random numbers. + :param offsets: The offsets to generate random numbers for. + """ + i1, i2, _, _ = randint4x(seed, offset, n_rounds) + u1 = uint_to_uniform_float(i1) + u2 = uint_to_uniform_float(i2) + n1, _ = pair_uniform_to_normal(u1, u2) + return n1 + + +@jit +def randn4x(seed, offset, n_rounds: tl.constexpr = N_ROUNDS_DEFAULT): + """ + Given a :code:`seed` scalar and an :code:`offset` block, + returns 4 blocks of random :code:`float32` in :math:`\\mathcal{N}(0, 1)`. + + :param seed: The seed for generating random numbers. + :param offsets: The offsets to generate random numbers for. + """ + u1, u2, u3, u4 = rand4x(seed, offset, n_rounds) + n1, n2 = pair_uniform_to_normal(u1, u2) + n3, n4 = pair_uniform_to_normal(u3, u4) + return n1, n2, n3, n4 diff --git a/lib/python3.12/site-packages/triton/language/semantic.py b/lib/python3.12/site-packages/triton/language/semantic.py new file mode 100644 index 0000000000000000000000000000000000000000..8e9f87b5ed1232eabd769e571a5f8430a4ce08f6 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/semantic.py @@ -0,0 +1,1796 @@ +from __future__ import annotations # remove after python 3.11 +import warnings + +from typing import List, Optional, Sequence, Tuple, TypeVar +import numbers + +from .._C.libtriton import ir +from . import core as tl +from . import math + +T = TypeVar('T') + + +class IncompatibleTypeErrorImpl(Exception): + + def __init__(self, type_a, type_b): + self.type_a = type_a + self.type_b = type_b + self.message = "invalid operands of type " + self.type_a.__repr__() + " and " + self.type_b.__repr__() + super(IncompatibleTypeErrorImpl, self).__init__(self.message) + + +# ===----------------------------------------------------------------------===## +# Programming Model +# ===----------------------------------------------------------------------===## + + +def program_id(axis: int, builder: ir.builder) -> tl.tensor: + if axis not in (0, 1, 2): + raise ValueError(f"program_id axis must be 0, 1, or 2 but got {axis}") + return tl.tensor(builder.create_get_program_id(axis), tl.int32) + + +def num_programs(axis: int, builder: ir.builder) -> tl.tensor: + if axis not in (0, 1, 2): + raise ValueError(f"num_programs axis must be 0, 1, or 2 but got {axis}") + return tl.tensor(builder.create_get_num_programs(axis), tl.int32) + + +# ===----------------------------------------------------------------------===// +# Implicit Casting Utilities +# ===----------------------------------------------------------------------===// + + +def integer_promote_impl(a_ty: tl.dtype, b_ty: tl.dtype) -> tl.dtype: + a_rank = a_ty.int_bitwidth + b_rank = b_ty.int_bitwidth + a_sn = a_ty.int_signedness + b_sn = b_ty.int_signedness + # Rules for signedness taken from "Usual arithmetic conversions" on + # https://en.cppreference.com/w/c/language/conversion. + if a_sn == b_sn: + return a_ty if a_rank > b_rank else b_ty + elif a_sn == tl.dtype.SIGNEDNESS.UNSIGNED: + return a_ty if a_rank >= b_rank else b_ty + elif b_sn == tl.dtype.SIGNEDNESS.UNSIGNED: + return b_ty if b_rank >= a_rank else a_ty + raise TypeError(f"unexpected signedness {a_sn} and {b_sn}") + + +def computation_type_impl(a_ty: tl.dtype, a_is_scalar: bool, b_ty: tl.dtype, b_is_scalar: bool, + div_or_mod: bool) -> tl.dtype: + # 0) For scalars we follow semantics similar to PyTorch, namely: + # - If the scalar is of a lower or equal kind (bool < uint < int < fp), + # it doesn't participate in the pomotion + if a_is_scalar != b_is_scalar: + scalar_ty, tensor_ty = (a_ty, b_ty) if a_is_scalar else (b_ty, a_ty) + if scalar_ty.kind().value <= tensor_ty.kind().value: + # Upcast because of 3) and 4) below! + if div_or_mod and (tensor_ty in (tl.float16, tl.bfloat16)): + return tl.float32 + return tensor_ty + + # 1) if one operand is double, the other is implicitly + # converted to double + if a_ty.is_fp64() or b_ty.is_fp64(): + return tl.float64 + # 2) if one operand is float, the other is implicitly + # converted to float + if a_ty.is_fp32() or b_ty.is_fp32(): + return tl.float32 + # 3 ) if one operand is half, the other is implicitly converted to half + # unless we're doing / or %, which do not exist natively in PTX for fp16. + # Supported PTX op: add, sub, mul, fma, neg, abs, min, max, tanh, ex2, setp + if a_ty.is_fp16() or b_ty.is_fp16(): + if div_or_mod: + return tl.float32 + else: + return tl.float16 + # 4) return bf16 only if both operands are of bf16 + if a_ty.is_bf16() or b_ty.is_bf16(): + if div_or_mod: + return tl.float32 + if a_ty.is_bf16() and b_ty.is_bf16(): + return tl.bfloat16 + return tl.float32 + # 5) return fp16 if operands are different fp8 + if a_ty.is_fp8() and b_ty.is_fp8(): + return a_ty if a_ty == b_ty else tl.float16 + if not a_ty.is_int() or not b_ty.is_int(): + raise TypeError(f"unexpected type {a_ty} and {b_ty}") + # 6 ) both operands are integer and undergo + # integer promotion + if div_or_mod and a_ty.int_signedness != b_ty.int_signedness: + raise TypeError("Cannot use /, #, or % with " + a_ty.__repr__() + " and " + b_ty.__repr__() + + " because they have different signedness;" + "this is unlikely to result in a useful answer. Cast them to the same signedness.") + return integer_promote_impl(a_ty, b_ty) + + +def to_tensor(x, builder, check_type: bool = True): + if isinstance(x, bool): + return tl.tensor(builder.get_int1(x), tl.int1) + # Note: compile-time const integers are represented by unsigned values + elif isinstance(x, int): + if -2**31 <= x < 2**31: + dtype = tl.int32 + elif 2**31 <= x < 2**32: + dtype = tl.uint32 + elif -2**63 <= x < 2**63: + dtype = tl.int64 + elif 2**63 <= x < 2**64: + dtype = tl.uint64 + else: + raise ValueError(f'Nonrepresentable integer {x}.') + return full((), x, dtype=dtype, builder=builder) + elif isinstance(x, float): + min_float32 = 2**-126 + max_float32 = (2 - 2**-23) * 2**127 + abs_x = __builtins__['abs'](x) + if abs_x == float("inf") or\ + abs_x == 0.0 or \ + x != x or \ + min_float32 <= abs_x <= max_float32: + dtype = tl.float32 + else: + dtype = tl.float64 + return full((), x, dtype=dtype, builder=builder) + + elif isinstance(x, tl.constexpr): + return to_tensor(x.value, builder) + elif isinstance(x, tl.tensor): + return x + if check_type: + raise TypeError(f"cannot convert {x} of type {type(x)} to tensor") + return x + + +# ===----------------------------------------------------------------------===// +# Binary Operators +# ===----------------------------------------------------------------------===// + + +def check_ptr_type_impl(type_a: tl.dtype, type_b: tl.dtype, allow_ptr_a: bool) -> None: + if type_a.is_ptr(): + if not allow_ptr_a: + raise IncompatibleTypeErrorImpl(type_a, type_b) + # T* + U* with T != U + if type_b.is_ptr() and (type_a != type_b): + raise IncompatibleTypeErrorImpl(type_a, type_b) + # T* + float + if type_b.is_floating(): + raise IncompatibleTypeErrorImpl(type_a, type_b) + + +def binary_op_type_checking_impl(lhs: tl.tensor | numbers.Number, rhs: tl.tensor | numbers.Number, builder: ir.builder, + allow_lhs_ptr=False, allow_rhs_ptr=False, arithmetic_check=True, + div_or_mod=False) -> Tuple[tl.tensor, tl.tensor]: + lhs_is_scalar = isinstance(lhs, numbers.Number) + rhs_is_scalar = isinstance(rhs, numbers.Number) + if lhs_is_scalar: + lhs_scalar = lhs + lhs = to_tensor(lhs, builder) + if rhs_is_scalar: + rhs_scalar = rhs + rhs = to_tensor(rhs, builder) + + # implicit typecasting + lhs_sca_ty = lhs.type.scalar + rhs_sca_ty = rhs.type.scalar + check_ptr_type_impl(lhs_sca_ty, rhs_sca_ty, allow_lhs_ptr) + check_ptr_type_impl(rhs_sca_ty, lhs_sca_ty, allow_rhs_ptr) + if arithmetic_check and not lhs_sca_ty.is_ptr() and not rhs_sca_ty.is_ptr(): + ret_sca_ty = computation_type_impl(lhs_sca_ty, lhs_is_scalar, rhs_sca_ty, rhs_is_scalar, div_or_mod) + if (lhs_is_scalar and lhs_scalar < 0 and ret_sca_ty.is_int_unsigned() + or rhs_is_scalar and rhs_scalar < 0 and ret_sca_ty.is_int_unsigned()): + raise ValueError("Cannot perform a binary operation between an unsigned tensor and a negative scalar. " + "Perform a explicit cast on one of them.") + lhs = full( + (), lhs_scalar, dtype=ret_sca_ty, builder=builder) if lhs_is_scalar else cast(lhs, ret_sca_ty, builder) + rhs = full( + (), rhs_scalar, dtype=ret_sca_ty, builder=builder) if rhs_is_scalar else cast(rhs, ret_sca_ty, builder) + + # implicit broadcasting + lhs, rhs = broadcast_impl_value(lhs, rhs, builder) + return lhs, rhs + + +def binary_op_sanitize_overflow_impl(lhs: tl.tensor, rhs: tl.tensor, builder: ir.builder, binary_op: callable): + if lhs.type.scalar.int_bitwidth >= 64 or not builder.options.sanitize_overflow: + return + lhs_sca_ty = lhs.type.scalar + rhs_sca_ty = rhs.type.scalar + assert lhs_sca_ty == rhs_sca_ty + assert lhs_sca_ty.is_int() + lhs = cast(lhs, tl.int64, builder) + rhs = cast(rhs, tl.int64, builder) + ret = binary_op(lhs, rhs, False, builder) + max_value = lhs_sca_ty.get_int_max_value() + max_value = tl.tensor(builder.get_int64(max_value), tl.int64) + min_value = lhs_sca_ty.get_int_min_value() + min_value = tl.tensor(builder.get_int64(min_value), tl.int64) + cond = and_(less_equal(ret, max_value, builder), greater_equal(ret, min_value, builder), builder) + msg = f"int{lhs_sca_ty.int_bitwidth} overflow detected for operation {binary_op.__name__}" + device_assert(cond, msg, builder) + + +def add(input: tl.tensor | numbers.Number, other: tl.tensor | numbers.Number, sanitize_overflow: bool, + builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder, True, True) + input_scalar_ty = input.type.scalar + other_scalar_ty = other.type.scalar + if input_scalar_ty.is_ptr() and other_scalar_ty.is_ptr(): + raise TypeError("cannot add pointers together") + + # offset + ptr + # ptr + offset + if other_scalar_ty.is_ptr() and not input_scalar_ty.is_ptr(): + input, other = other, input + input_scalar_ty = input.type.scalar + other_scalar_ty = other.type.scalar + if input_scalar_ty.is_ptr(): + return tl.tensor(builder.create_addptr(input.handle, other.handle), input.type) + # float + float + elif input_scalar_ty.is_floating(): + return tl.tensor(builder.create_fadd(input.handle, other.handle), input.type) + # int + int + elif input_scalar_ty.is_int(): + if sanitize_overflow: + binary_op_sanitize_overflow_impl(input, other, builder, add) + return tl.tensor(builder.create_add(input.handle, other.handle), input.type) + raise TypeError(f"unexpected type {input_scalar_ty}") + + +def sub(input: tl.tensor | numbers.Number, other: tl.tensor | numbers.Number, sanitize_overflow: bool, + builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder, True, False) + scalar_ty = input.type.scalar + # ptr - offset + if scalar_ty.is_ptr(): + return tl.tensor(builder.create_addptr(input.handle, minus(other, builder).handle), input.type) + # float - float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fsub(input.handle, other.handle), input.type) + # int - int + elif scalar_ty.is_int(): + if sanitize_overflow: + binary_op_sanitize_overflow_impl(input, other, builder, sub) + return tl.tensor(builder.create_sub(input.handle, other.handle), input.type) + raise TypeError(f"unexpected type {scalar_ty}") + + +def mul(input: tl.tensor | numbers.Number, other: tl.tensor | numbers.Number, sanitize_overflow: bool, + builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder) + scalar_ty = input.type.scalar + # float * float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fmul(input.handle, other.handle), input.type) + # int * int + elif scalar_ty.is_int(): + if sanitize_overflow: + binary_op_sanitize_overflow_impl(input, other, builder, mul) + return tl.tensor(builder.create_mul(input.handle, other.handle), input.type) + raise TypeError(f"unexpected type {scalar_ty}") + + +def truediv(input: tl.tensor | numbers.Number, other: tl.tensor | numbers.Number, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder, False, False, True, True) + input_scalar_ty = input.type.scalar + other_scalar_ty = other.type.scalar + # float / int + if input_scalar_ty.is_floating() and other_scalar_ty.is_int(): + other = cast(other, input_scalar_ty, builder) + # int / float + elif input_scalar_ty.is_int() and other_scalar_ty.is_floating(): + input = cast(input, other_scalar_ty, builder) + # int / int (cast to tl.float32) + elif input_scalar_ty.is_int() and other_scalar_ty.is_int(): + input = cast(input, tl.float32, builder) + other = cast(other, tl.float32, builder) + # float / float (cast to the highest exponent type) + elif input_scalar_ty.is_floating() and other_scalar_ty.is_floating(): + if input_scalar_ty.fp_mantissa_width > other_scalar_ty.fp_mantissa_width: + other = cast(other, input_scalar_ty, builder) + else: + input = cast(input, other_scalar_ty, builder) + # unreachable + else: + raise TypeError(f"unexpected type {input_scalar_ty}") + return tl.tensor(builder.create_fdiv(input.handle, other.handle), input.type) + + +def floordiv(input: tl.tensor | numbers.Number, other: tl.tensor | numbers.Number, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder, False, False, True, True) + input_scalar_ty = input.type.scalar + other_scalar_ty = other.type.scalar + if input_scalar_ty.is_int() and other_scalar_ty.is_int(): + ret_ty = integer_promote_impl(input_scalar_ty, other_scalar_ty) + input = cast(input, ret_ty, builder) + other = cast(other, ret_ty, builder) + if ret_ty.is_int_signed(): + return tl.tensor(builder.create_sdiv(input.handle, other.handle), input.type) + else: + return tl.tensor(builder.create_udiv(input.handle, other.handle), input.type) + raise TypeError(f"unexpected type {input_scalar_ty}") + + +def fdiv(input: tl.tensor | numbers.Number, other: tl.tensor | numbers.Number, ieee_rounding: bool, + builder: ir.builder) -> tl.tensor: + input_scalar_ty = input.type.scalar + other_scalar_ty = other.type.scalar + if not input_scalar_ty.is_floating() or not other_scalar_ty.is_floating(): + raise TypeError("both operands of fdiv must have floating scalar type") + input, other = binary_op_type_checking_impl(input, other, builder, False, False, False, True) + ret = builder.create_fdiv(input.handle, other.handle) + return tl.tensor(ret, input.type) + + +def mod(input: tl.tensor | numbers.Number, other: tl.tensor | numbers.Number, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder, False, False, True, True) + scalar_ty = input.type.scalar + other_scalar_ty = other.type.scalar + # float % float + if scalar_ty.is_floating(): + # input - input.div(other, rounding_mode="floor") * other + floor = math.floor(fdiv(input, other, False, builder), _builder=builder) + ret = sub(input, mul(floor, other, True, builder), True, builder) + return ret + # % int + elif scalar_ty.is_int(): + if scalar_ty.int_signedness != other_scalar_ty.int_signedness: + raise TypeError("Cannot mod " + scalar_ty.__repr__() + " by " + other_scalar_ty.__repr__() + " " + "because they have different signedness;" + "this is unlikely to result in a useful answer. Cast them to the same signedness.") + if scalar_ty.is_int_signed(): + return tl.tensor(builder.create_srem(input.handle, other.handle), input.type) + else: + return tl.tensor(builder.create_urem(input.handle, other.handle), input.type) + raise TypeError(f"unexpected type {scalar_ty}") + + +############## +# other arithmetic ops +############## + + +def minimum(x: tl.tensor, y: tl.tensor, propagate_nan: tl.PropagateNan, builder: ir.builder): + x, y = binary_op_type_checking_impl(x, y, builder) + dtype = x.dtype + if dtype.is_floating(): + if propagate_nan == tl.PropagateNan.ALL: + return tl.tensor(builder.create_minimumf(x.handle, y.handle), x.type) + elif propagate_nan == tl.PropagateNan.NONE: + return tl.tensor(builder.create_minnumf(x.handle, y.handle), x.type) + else: + raise ValueError(f"Unexpected propagate_nan {propagate_nan}") + elif dtype.is_int_signed(): + return tl.tensor(builder.create_minsi(x.handle, y.handle), x.type) + elif dtype.is_int_unsigned(): + return tl.tensor(builder.create_minui(x.handle, y.handle), x.type) + else: + raise TypeError(f"Unexpected dtype {dtype}") + + +def maximum(x: tl.tensor, y: tl.tensor, propagate_nan: tl.PropagateNan, builder: ir.builder): + x, y = binary_op_type_checking_impl(x, y, builder) + dtype = x.dtype + if dtype.is_floating(): + if propagate_nan == tl.PropagateNan.ALL: + return tl.tensor(builder.create_maximumf(x.handle, y.handle), x.type) + elif propagate_nan == tl.PropagateNan.NONE: + return tl.tensor(builder.create_maxnumf(x.handle, y.handle), x.type) + else: + raise ValueError(f"Unexpected propagate_nan {propagate_nan}") + elif dtype.is_int_signed(): + return tl.tensor(builder.create_maxsi(x.handle, y.handle), x.type) + elif dtype.is_int_unsigned(): + return tl.tensor(builder.create_maxui(x.handle, y.handle), x.type) + else: + raise TypeError(f"Unexpected dtype {dtype}") + + +def clamp(x: tl.tensor, min: tl.tensor, max: tl.tensor, propagate_nan: tl.PropagateNan, builder: ir.builder): + min, max = binary_op_type_checking_impl(min, max, builder) + x, min = binary_op_type_checking_impl(x, min, builder) + x, max = binary_op_type_checking_impl(x, max, builder) + + dtype = x.dtype + if dtype.is_floating(): + return tl.tensor(builder.create_clampf(x.handle, min.handle, max.handle, propagate_nan), x.type) + else: + raise TypeError(f"Unexpected dtype {dtype}. Only floating point clamp is supported") + + +############## +# bitwise ops +############## + + +def bitwise_op_type_checking_impl(input: tl.tensor, other: tl.tensor, + builder: ir.builder) -> Tuple[tl.tensor, tl.tensor]: + input, other = binary_op_type_checking_impl(input, other, builder) + input_sca_ty = input.type.scalar + other_sca_ty = other.type.scalar + if not input_sca_ty.is_int() or not other_sca_ty.is_int(): + raise IncompatibleTypeErrorImpl(input_sca_ty, other_sca_ty) + ret_sca_ty = integer_promote_impl(input_sca_ty, other_sca_ty) + if ret_sca_ty != input_sca_ty: + input = cast(input, ret_sca_ty, builder) + if ret_sca_ty != other_sca_ty: + other = cast(other, ret_sca_ty, builder) + return input, other + + +def and_(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = bitwise_op_type_checking_impl(input, other, builder) + return tl.tensor(builder.create_and(input.handle, other.handle), input.type) + + +def or_(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = bitwise_op_type_checking_impl(input, other, builder) + return tl.tensor(builder.create_or(input.handle, other.handle), input.type) + + +def xor_(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = bitwise_op_type_checking_impl(input, other, builder) + return tl.tensor(builder.create_xor(input.handle, other.handle), input.type) + + +def logical_and(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + if not input.type.is_int1(): + input = bitcast(input, tl.dtype("int1"), builder) + if not other.type.is_int1(): + other = bitcast(other, tl.dtype("int1"), builder) + return and_(input, other, builder) + + +def logical_or(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + if not input.type.is_int1(): + input = bitcast(input, tl.dtype("int1"), builder) + if not other.type.is_int1(): + other = bitcast(other, tl.dtype("int1"), builder) + return or_(input, other, builder) + + +def not_(input: tl.tensor, builder: ir.builder): + if not input.type.is_int1(): + input = bitcast(input, tl.dtype("int1"), builder) + return invert(input, builder) + + +def lshr(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = bitwise_op_type_checking_impl(input, other, builder) + return tl.tensor(builder.create_lshr(input.handle, other.handle), input.type) + + +def ashr(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = bitwise_op_type_checking_impl(input, other, builder) + return tl.tensor(builder.create_ashr(input.handle, other.handle), input.type) + + +def shl(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = bitwise_op_type_checking_impl(input, other, builder) + return tl.tensor(builder.create_shl(input.handle, other.handle), input.type) + + +# ===----------------------------------------------------------------------===// +# Unary Operators +# ===----------------------------------------------------------------------===// + + +def plus(input: tl.tensor) -> tl.tensor: + return input + + +def minus(input: tl.tensor, builder: ir.builder) -> tl.tensor: + input_sca_ty = input.type.scalar + if input_sca_ty.is_ptr(): + raise ValueError("wrong type argument to unary minus (" + input_sca_ty.__repr__() + ")") + _0 = tl.tensor(builder.get_null_value(input_sca_ty.to_ir(builder)), input_sca_ty) + return sub(_0, input, True, builder) + + +def invert(input: tl.tensor, builder: tl.tensor) -> tl.tensor: + input_sca_ty = input.type.scalar + if input_sca_ty.is_ptr() or input_sca_ty.is_floating(): + raise ValueError("wrong type argument to unary invert (" + input_sca_ty.__repr__() + ")") + _1 = tl.tensor(builder.get_all_ones_value(input_sca_ty.to_ir(builder)), input_sca_ty) + return xor_(input, _1, builder) + + +# ===----------------------------------------------------------------------===// +# Comparison Operators +# ===----------------------------------------------------------------------===// +def _bool_like(v: tl.tensor) -> tl.block_type: + if not v.type.is_block(): + return tl.int1 + shape = v.type.shape + return tl.block_type(tl.int1, shape) + + +def greater_than(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder) + scalar_ty = input.type.scalar + # float > float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fcmpOGT(input.handle, other.handle), _bool_like(input)) + # > int + elif scalar_ty.is_int(): + if scalar_ty.is_int_signed(): + return tl.tensor(builder.create_icmpSGT(input.handle, other.handle), _bool_like(input)) + else: + return tl.tensor(builder.create_icmpUGT(input.handle, other.handle), _bool_like(input)) + raise TypeError(f"unexpected type {scalar_ty}") + + +def greater_equal(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder) + scalar_ty = input.type.scalar + # float >= float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fcmpOGE(input.handle, other.handle), _bool_like(input)) + # >= int + elif scalar_ty.is_int(): + if scalar_ty.is_int_signed(): + return tl.tensor(builder.create_icmpSGE(input.handle, other.handle), _bool_like(input)) + else: + return tl.tensor(builder.create_icmpUGE(input.handle, other.handle), _bool_like(input)) + raise TypeError(f"unexpected type {scalar_ty}") + + +def less_than(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder) + scalar_ty = input.type.scalar + # float < float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fcmpOLT(input.handle, other.handle), _bool_like(input)) + # < int + elif scalar_ty.is_int(): + if scalar_ty.is_int_signed(): + return tl.tensor(builder.create_icmpSLT(input.handle, other.handle), _bool_like(input)) + else: + return tl.tensor(builder.create_icmpULT(input.handle, other.handle), _bool_like(input)) + raise TypeError(f"unexpected type {scalar_ty}") + + +def less_equal(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder) + scalar_ty = input.type.scalar + # float < float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fcmpOLE(input.handle, other.handle), _bool_like(input)) + # < int + elif scalar_ty.is_int(): + if scalar_ty.is_int_signed(): + return tl.tensor(builder.create_icmpSLE(input.handle, other.handle), _bool_like(input)) + else: + return tl.tensor(builder.create_icmpULE(input.handle, other.handle), _bool_like(input)) + raise TypeError(f"unexpected type {scalar_ty}") + + +def equal(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder) + scalar_ty = input.type.scalar + # float == float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fcmpOEQ(input.handle, other.handle), _bool_like(input)) + # == int + elif scalar_ty.is_int(): + return tl.tensor(builder.create_icmpEQ(input.handle, other.handle), _bool_like(input)) + raise TypeError(f"unexpected type {scalar_ty}") + + +def not_equal(input: tl.tensor, other: tl.tensor, builder: ir.builder) -> tl.tensor: + input, other = binary_op_type_checking_impl(input, other, builder) + scalar_ty = input.type.scalar + # float == float + if scalar_ty.is_floating(): + return tl.tensor(builder.create_fcmpUNE(input.handle, other.handle), _bool_like(input)) + # == int + elif scalar_ty.is_int(): + return tl.tensor(builder.create_icmpNE(input.handle, other.handle), _bool_like(input)) + raise TypeError(f"unexpected type {scalar_ty}") + + +# ===----------------------------------------------------------------------===// +# Block Creation +# ===----------------------------------------------------------------------===// + + +def arange(start: int, end: int, builder: ir.builder) -> tl.tensor: + if not isinstance(start, int) or not isinstance(end, int): + raise ValueError("arange's arguments must be of type tl.constexpr") + is_start_int64 = bool(start >> 32) + is_end_int64 = bool(end >> 32) + if is_start_int64 or is_end_int64: + raise ValueError("arange must fit in int32") + if end <= start: + raise ValueError("arange's end argument must be greater than the start argument") + range = end - start + if (range & (range - 1)) != 0: + raise ValueError("arange's range must be a power of 2") + shape = [range] + ret_ty = tl.block_type(tl.int32, shape) + return tl.tensor(builder.create_make_range(start, end), ret_ty) + + +def full(shape: List[int], value, dtype: tl.dtype, builder: ir.builder) -> tl.tensor: + if isinstance(value, tl.tensor): + assert value.numel.value == 1, "only accepts size-1 tensor" + value = cast(value, dtype, builder) + else: + # scalar + if dtype is None: + raise ValueError("dtype must be specified when value is not a tensor") + if value == 0: + value = builder.get_null_value(dtype.to_ir(builder)) + else: + get_value_fn = getattr(builder, f"get_{dtype.name}") + value = get_value_fn(value) + value = tl.tensor(value, dtype) + + return splat(value, shape, builder) + + +# ===----------------------------------------------------------------------===// +# Shape Manipulation +# ===----------------------------------------------------------------------===// + + +def splat(value: tl.tensor, shape: List[int], builder: ir.builder) -> tl.tensor: + assert not value.type.is_block(), "Cannot splat a block tensor" + if len(shape) == 0: + return value + ret_ty = tl.block_type(value.dtype, shape) + return tl.tensor(builder.create_splat(value.handle, shape), ret_ty) + + +def reshape(input: tl.tensor, dst_shape: List[int], can_reorder: bool, builder: ir.builder) -> tl.tensor: + numel = 1 + for s in dst_shape: + numel *= s + if input.type.numel != numel: + raise ValueError("reshape() cannot change total number of elements in tensor") + ret_ty = tl.block_type(input.type.scalar, dst_shape) + return tl.tensor(builder.create_reshape(input.handle, dst_shape, can_reorder), ret_ty) + + +def expand_dims(input: tl.tensor, axis: int, builder: ir.builder) -> tl.tensor: + dst_shape = [tl._constexpr_to_value(x) for x in input.shape] + dst_shape.insert(axis, 1) + + if not input.type.is_block(): + return splat(input, shape=dst_shape, builder=builder) + + ret_ty = tl.block_type(input.type.scalar, dst_shape) + return tl.tensor(builder.create_expand_dims(input.handle, axis), ret_ty) + + +def cat(lhs: tl.tensor, rhs: tl.tensor, can_reorder: bool, builder: ir.builder) -> tl.tensor: + assert can_reorder, "current implementation of `cat` always may reorder elements" + assert len(lhs.shape) == 1 + ret_type = tl.block_type(lhs.type.scalar, [lhs.shape[0] + rhs.shape[0]]) + return tl.tensor(builder.create_cat(lhs.handle, rhs.handle), ret_type) + + +def join(a: tl.tensor, b: tl.tensor, builder: ir.builder) -> tl.tensor: + a, b = broadcast_impl_value(a, b, builder) + + # The IR can't handle joining two scalars, so upcast them to 1D tensors, + # then downcast the result. + was_rank_1 = a.shape == [] + if was_rank_1: + a = expand_dims(a, 0, builder) + b = expand_dims(b, 0, builder) + + if isinstance(a.shape[-1], tl.constexpr): + two = tl.constexpr(2) + else: + two = 2 + new_shape = a.shape + [two] + + ret_type = tl.block_type(a.type.scalar, new_shape) + ret = tl.tensor(builder.create_join(a.handle, b.handle), ret_type) + + if was_rank_1: + ret = reshape(ret, [2], can_reorder=False, builder=builder) + + return ret + + +def split(a: tl.tensor, builder: ir.builder) -> Tuple[tl.tensor, tl.tensor]: + assert (len(a.shape) > 0) + assert (tl._constexpr_to_value(a.shape[-1]) == 2) + + new_shape = a.shape[:-1] + ret_type = tl.block_type(a.type.scalar, new_shape) + outLHS, outRHS = builder.create_split(a.handle) + return ( + tl.tensor(outLHS, ret_type), + tl.tensor(outRHS, ret_type), + ) + + +def permute(input: tl.tensor, dims: Tuple[int], builder: ir.builder) -> tl.tensor: + if len(input.shape) != len(dims): + raise ValueError("permute dims must have the same length as input shape") + if sorted(tl._constexpr_to_value(d) for d in dims) != list(range(len(dims))): + raise ValueError(f"permute dims must be a permutation of 0, 1, ..., n-1, but were {dims}") + + ret_type = tl.block_type(input.type.scalar, [input.shape[d] for d in dims]) + return tl.tensor(builder.create_trans(input.handle, dims), ret_type) + + +def broadcast_impl_shape(input: tl.tensor, shape: List[int], builder: ir.builder) -> tl.tensor: + if not input.type.is_block(): + ret_ty = tl.block_type(input.type, shape) + return tl.tensor(builder.create_splat(input.handle, shape), ret_ty) + src_shape = input.type.get_block_shapes() + if len(src_shape) != len(shape): + raise ValueError(f"Cannot broadcast, rank mismatch: {src_shape}, {shape}") + if shape == src_shape: + return input + for i, item in enumerate(src_shape): + if shape[i] != item and item != 1: + raise ValueError(f"Cannot broadcast, the expanded size of the tensor ({shape[i]})" + f" must match the existing size ({item}) at non-singleton dimension" + f" {i}: {src_shape}, {shape}") + ret_ty = tl.block_type(input.type.scalar, shape) + return tl.tensor(builder.create_broadcast(input.handle, shape), ret_ty) + + +def broadcast_impl_value(lhs: tl.tensor, rhs: tl.tensor, builder: ir.builder) -> tl.tensor: + lhs_ty = lhs.type + rhs_ty = rhs.type + + # make_shape_compatible(block, scalar) + if lhs_ty.is_block() and not rhs_ty.is_block(): + rhs_ty = tl.block_type(rhs_ty.scalar, lhs_ty.shape) + rhs = tl.tensor(builder.create_splat(rhs.handle, lhs_ty.get_block_shapes()), rhs_ty) + # make_shape_compatible(scalar, block) + elif not lhs_ty.is_block() and rhs_ty.is_block(): + lhs_ty = tl.block_type(lhs_ty.scalar, rhs_ty.shape) + lhs = tl.tensor(builder.create_splat(lhs.handle, rhs_ty.get_block_shapes()), lhs_ty) + # make_shape_compatible(block, block) + elif lhs_ty.is_block() and rhs_ty.is_block(): + lhs_shape = lhs_ty.get_block_shapes() + rhs_shape = rhs_ty.get_block_shapes() + + if len(lhs_shape) < len(rhs_shape): + # Add new axes to lhs + for _ in range(len(lhs_shape), len(rhs_shape)): + lhs = tl.tensor(builder.create_expand_dims(lhs.handle, 0), + tl.block_type(lhs_ty.scalar, [1] + lhs_shape)) + lhs_ty = lhs.type + lhs_shape = lhs_ty.get_block_shapes() + elif len(rhs_shape) < len(lhs_shape): + # Add new axes to rhs + for _ in range(len(rhs_shape), len(lhs_shape)): + rhs = tl.tensor(builder.create_expand_dims(rhs.handle, 0), + tl.block_type(rhs_ty.scalar, [1] + rhs_shape)) + rhs_ty = rhs.type + rhs_shape = rhs_ty.get_block_shapes() + assert len(rhs_shape) == len(lhs_shape) + + ret_shape = [] + for i, left in enumerate(lhs_shape): + right = rhs_shape[i] + if left == 1: + ret_shape.append(right) + elif (right == 1) or (right == left): + ret_shape.append(left) + else: + raise ValueError("Cannot make_shape_compatible: incompatible dimensions " + "at index " + str(i) + ": " + str(left) + " and " + str(right)) + if lhs_shape != ret_shape: + ret_ty = tl.block_type(lhs_ty.scalar, ret_shape) + lhs = tl.tensor(builder.create_broadcast(lhs.handle, ret_shape), ret_ty) + if rhs_shape != ret_shape: + ret_ty = tl.block_type(rhs_ty.scalar, ret_shape) + rhs = tl.tensor(builder.create_broadcast(rhs.handle, ret_shape), ret_ty) + # (scalar, scalar) => returns original blocks + return lhs, rhs + + +####### +# cast +####### + + +def _str_to_rounding_mode(rounding_mode: Optional[str]): + if rounding_mode is None: + return None + if rounding_mode == 'rtne': + return ir.ROUNDING_MODE.RTNE + if rounding_mode == 'rtz': + return ir.ROUNDING_MODE.RTZ + raise ValueError(f"Invalid rounding mode: {rounding_mode}. Supported rounding modes are 'rtne' and 'rtz'.") + + +def bitcast(input: tl.tensor, dst_ty: tl.dtype, builder: ir.builder) -> tl.tensor: + src_ty = input.type + if src_ty.is_block(): + dst_ty = tl.block_type(dst_ty.scalar, input.type.get_block_shapes()) + if src_ty == dst_ty: + return input + src_sca_ty = src_ty.scalar + dst_sca_ty = dst_ty.scalar + if src_sca_ty.is_ptr() or dst_sca_ty.is_ptr(): + return cast(input, dst_ty, builder) + # Bitcast + src_bits = src_sca_ty.primitive_bitwidth + dst_bits = dst_sca_ty.primitive_bitwidth + if src_bits != dst_bits: + raise ValueError("Cannot bitcast data-type of size " + str(src_bits) + " to " + "data-type of size " + str(dst_bits)) + return tl.tensor(builder.create_bitcast(input.handle, dst_ty.to_ir(builder)), dst_ty) + + +def cast(input: tl.tensor, dst_ty: tl.dtype, builder: ir.builder, + fp_downcast_rounding: Optional[str] = None) -> tl.tensor: + src_ty = input.type + if isinstance(dst_ty, tl.constexpr): + dst_ty = dst_ty.value + if isinstance(fp_downcast_rounding, tl.constexpr): + fp_downcast_rounding = fp_downcast_rounding.value + if src_ty.is_block(): + dst_ty = tl.block_type(dst_ty.scalar, input.type.get_block_shapes()) + if src_ty == dst_ty: + return input + + src_sca_ty = src_ty.scalar + dst_sca_ty = dst_ty.scalar + + # For fp downcasting default rounding mode should be RTNE, for all other conversions it should + # not be set + fp_downcast_rounding = _str_to_rounding_mode(fp_downcast_rounding) + use_custom_rounding = False + if dst_sca_ty.is_floating() and src_sca_ty.is_floating( + ) and dst_sca_ty.primitive_bitwidth < src_sca_ty.primitive_bitwidth: + if fp_downcast_rounding is None: fp_downcast_rounding = ir.ROUNDING_MODE.RTNE + elif fp_downcast_rounding != ir.ROUNDING_MODE.RTNE: use_custom_rounding = True + else: + if fp_downcast_rounding is not None: + raise ValueError("fp_downcast_rounding should be set only for truncating fp conversions. " + "Source scalar type is " + str(src_sca_ty) + " and destination type is " + str(dst_sca_ty)) + + if (src_sca_ty.is_fp8e4b15() or dst_sca_ty.is_fp8e4b15()): + assert builder.codegen_fns.get( + "convert_custom_types") is not None, "target doesn't provide conversion for this type." + return builder.codegen_fns["convert_custom_types"](input, dst_ty, fp_downcast_rounding, _builder=builder) + # Casting with customized floating types involved: fp8 <=> bf16, fp16, fp32, fp64 + # and non-default rounding modes for downcasting + if (src_sca_ty.is_fp8() and dst_sca_ty.is_floating()) or \ + (src_sca_ty.is_floating() and dst_sca_ty.is_fp8()) or \ + use_custom_rounding: + return tl.tensor(builder.create_fp_to_fp(input.handle, dst_ty.to_ir(builder), fp_downcast_rounding), dst_ty) + + # bf16 <=> (not fp32) + if (src_sca_ty.is_fp16() and not dst_sca_ty.is_fp32()) or \ + (src_sca_ty.is_bf16() and not dst_sca_ty.is_fp32()): + return cast(cast(input, tl.float32, builder), dst_sca_ty, builder) + + # Standard floating types' casting: truncation + # fp64 => fp32, fp16, bf16 + # fp32 => fp16, bf16 + truncate_fp = src_sca_ty.is_floating() and \ + dst_sca_ty.is_floating() and \ + src_sca_ty.primitive_bitwidth > dst_sca_ty.primitive_bitwidth + if truncate_fp: + return tl.tensor(builder.create_fp_trunc(input.handle, dst_ty.to_ir(builder)), dst_ty) + + # Standard floating types' casting: extension + # fp32 => fp64 + # fp16 => fp32, fp64 + # bf16 => fp32, fp64 + ext_fp = src_sca_ty.is_floating() and \ + dst_sca_ty.is_floating() and \ + src_sca_ty.primitive_bitwidth < dst_sca_ty.primitive_bitwidth + if ext_fp: + return tl.tensor(builder.create_fp_ext(input.handle, dst_ty.to_ir(builder)), dst_ty) + + # Casting between integer types + if src_sca_ty.is_int() and dst_sca_ty.is_int() and \ + (src_sca_ty.int_bitwidth != dst_sca_ty.int_bitwidth or src_sca_ty.int_signedness != dst_sca_ty.int_signedness): + sign_extend = src_sca_ty.is_int_signed() and not src_sca_ty.is_bool() + if dst_sca_ty.is_bool(): + ty = input.dtype.to_ir(builder) + _0 = tl.tensor(builder.get_null_value(ty), input.dtype) + return not_equal(input, _0, builder) + else: + return tl.tensor(builder.create_int_cast(input.handle, dst_ty.to_ir(builder), sign_extend), dst_ty) + + # Casting standard floating types to integer types + if src_sca_ty.is_standard_floating() and dst_sca_ty.is_int(): + if dst_sca_ty.is_bool(): + ty = input.dtype.to_ir(builder) + _0 = tl.tensor(builder.get_null_value(ty), input.dtype) + return not_equal(input, _0, builder) + elif dst_sca_ty.is_int_signed(): + return tl.tensor(builder.create_fp_to_si(input.handle, dst_ty.to_ir(builder)), dst_ty) + else: + return tl.tensor(builder.create_fp_to_ui(input.handle, dst_ty.to_ir(builder)), dst_ty) + + # Casting integer types to standard floating types + if src_sca_ty.is_int() and dst_sca_ty.is_standard_floating(): + if src_sca_ty.is_bool() or not src_sca_ty.is_int_signed(): + return tl.tensor(builder.create_ui_to_fp(input.handle, dst_ty.to_ir(builder)), dst_ty) + else: + return tl.tensor(builder.create_si_to_fp(input.handle, dst_ty.to_ir(builder)), dst_ty) + + # Casting pointer types to integer types + if src_sca_ty.is_ptr() and dst_sca_ty.is_int(): + bitwidth = dst_sca_ty.int_bitwidth + if bitwidth == 64: + return tl.tensor(builder.create_ptr_to_int(input.handle, dst_ty.to_ir(builder)), dst_ty) + if bitwidth == 1: + return not_equal(cast(input, tl.int64, builder), tl.tensor(builder.get_int64(0), tl.int64), builder) + + # Casting integer types to pointer types + if src_sca_ty.is_int() and dst_sca_ty.is_ptr(): + return tl.tensor(builder.create_int_to_ptr(input.handle, dst_ty.to_ir(builder)), dst_ty) + + # Casting pointer types to pointer types + if src_sca_ty.is_ptr() and dst_sca_ty.is_ptr(): + return tl.tensor(builder.create_bitcast(input.handle, dst_ty.to_ir(builder)), dst_ty) + + assert False, f'cannot cast {input} to {dst_ty}' + + +# ===----------------------------------------------------------------------===// +# Memory Operators +# ===----------------------------------------------------------------------===// + + +def _str_to_load_cache_modifier(cache_modifier): + cache = ir.CACHE_MODIFIER.NONE # default + if cache_modifier: + if cache_modifier == ".ca": + cache = ir.CACHE_MODIFIER.CA + elif cache_modifier == ".cg": + cache = ir.CACHE_MODIFIER.CG + elif cache_modifier == ".cv": + cache = ir.CACHE_MODIFIER.CV + else: + raise ValueError(f"Cache modifier {cache_modifier} not supported") + return cache + + +def _str_to_store_cache_modifier(cache_modifier): + cache = ir.CACHE_MODIFIER.NONE # default + if cache_modifier: + if cache_modifier == ".wb": + cache = ir.CACHE_MODIFIER.WB + elif cache_modifier == ".cg": + cache = ir.CACHE_MODIFIER.CG + elif cache_modifier == ".cs": + cache = ir.CACHE_MODIFIER.CS + elif cache_modifier == ".wt": + cache = ir.CACHE_MODIFIER.WT + else: + raise ValueError(f"Cache modifier {cache_modifier} not supported") + return cache + + +def _str_to_eviction_policy(eviction_policy): + eviction = ir.EVICTION_POLICY.NORMAL # default + if eviction_policy: + if eviction_policy == "evict_last": + eviction = ir.EVICTION_POLICY.EVICT_LAST + elif eviction_policy == "evict_first": + eviction = ir.EVICTION_POLICY.EVICT_FIRST + else: + raise ValueError(f"Eviction policy {eviction_policy} not supported") + return eviction + + +def _str_to_padding_option(padding_option): + padding = None # default + if padding_option: + if padding_option == "zero": + padding = ir.PADDING_OPTION.PAD_ZERO + elif padding_option == "nan": + padding = ir.PADDING_OPTION.PAD_NAN + else: + raise ValueError(f"Padding option {padding_option} not supported") + return padding + + +def _str_to_sem(sem_option): + sem = ir.MEM_SEMANTIC.ACQUIRE_RELEASE + if sem_option: + if sem_option == "acquire": + sem = ir.MEM_SEMANTIC.ACQUIRE + elif sem_option == "release": + sem = ir.MEM_SEMANTIC.RELEASE + elif sem_option == "acq_rel": + sem = ir.MEM_SEMANTIC.ACQUIRE_RELEASE + elif sem_option == "relaxed": + sem = ir.MEM_SEMANTIC.RELAXED + else: + raise ValueError(f"Memory semantic {sem_option} not supported") + return sem + + +def _str_to_scope(scope_option): + scope = ir.MEM_SYNC_SCOPE.GPU + if scope_option: + if scope_option == "gpu": + scope = ir.MEM_SYNC_SCOPE.GPU + elif scope_option == "cta": + scope = ir.MEM_SYNC_SCOPE.CTA + elif scope_option == "sys": + scope = ir.MEM_SYNC_SCOPE.SYSTEM + else: + raise ValueError(f"Memory semantic {scope_option} not supported") + return scope + + +def _canonicalize_boundary_check(boundary_check, block_shape): + if boundary_check: + if not hasattr(boundary_check, "__iter__"): + boundary_check = [boundary_check] + boundary_check = [elem.value if isinstance(elem, tl.constexpr) else elem for elem in boundary_check] + for dim in boundary_check: + assert isinstance(dim, int) and 0 <= dim < len(block_shape) + assert len(boundary_check) > 0 + assert len(boundary_check) == len(set(boundary_check)), "Duplicate dimension in `boundary_check`" + return sorted(boundary_check) + return () + + +def _load_block_pointer(ptr, mask, other, boundary_check, padding, cache, eviction, is_volatile, builder): + # Load by a block pointer: `pointer_type>` + # Block pointer can not have `mask` and `other` arguments + if mask is not None or other is not None: + raise ValueError("`mask` and `other` arguments cannot be specified for loading block pointers") + + elt_ty = ptr.type.element_ty.element_ty + assert elt_ty != tl.int1, "`tl.int1` should be rewrited in `tl.make_block_ptr`" + if elt_ty.is_int() and padding == ir.PADDING_OPTION.PAD_NAN: + raise ValueError("Padding option `nan` is not supported for integer block pointers") + + # `dst_ty` is de-referenced type of the pointer type + dst_ty = ptr.type.element_ty + + # Check `boundary_check` argument + boundary_check = _canonicalize_boundary_check(boundary_check, dst_ty.get_block_shapes()) + + # Build IR + return tl.tensor( + builder.create_tensor_pointer_load(ptr.handle, boundary_check, padding, cache, eviction, is_volatile), dst_ty) + + +def _load_legacy(ptr, mask, other, boundary_check, padding, cache, eviction, is_volatile, builder): + # Load by a tensor of pointers or a pointer of scalar: `block_type>` or `pointer_type<>` + if not ptr.type.scalar.is_ptr(): + raise ValueError(f"Unsupported ptr type {ptr.type.__repr__()} in `tl.load`") + + # Check `mask`, `other`, `boundary_check`, and `padding` arguments + if mask is None and other is not None: + raise ValueError("`other` cannot be provided without `mask`") + if padding or boundary_check: + raise ValueError("`padding_option` or `boundary_check` argument is not supported for loading a tensor of" + "pointers or loading a scalar. Because the compiler does not know the boundary; please " + "use block pointers (defined by `make_block_ptr`) instead") + + # For a pointer of scalar, check the type of `mask` and `other` + if not ptr.type.is_block(): + if mask and mask.type.is_block(): + raise ValueError("Mask argument cannot be block type if pointer argument is not a block") + if other and other.type.is_block(): + raise ValueError("Other argument cannot be block type if pointer argument is not a block") + + # Make `mask` and `other` into the same shape as `ptr` + if ptr.type.is_block(): + if mask is not None: + mask = broadcast_impl_shape(mask, ptr.type.get_block_shapes(), builder) + if other is not None: + other = broadcast_impl_shape(other, ptr.type.get_block_shapes(), builder) + + # Get `pointer_type` and `elt_ty` + ptr_ty = ptr.type.scalar + elt_ty = ptr_ty.element_ty + + # Treat `pointer_type` as `pointer_type` + is_bool = elt_ty == tl.int1 + if is_bool: + elt_ty = tl.int8 + ptr_ty = tl.pointer_type(elt_ty, ptr_ty.address_space) + ptr = cast(ptr, ptr_ty, builder) + + # Cast `other` into `elt_ty` type + if other is not None: + other = cast(other, elt_ty, builder) + + # Create loaded result type `dst_ty` + if ptr.type.is_block(): + shape = ptr.type.get_block_shapes() + dst_ty = tl.block_type(elt_ty, shape) + else: + # Load by de-referencing the pointer of scalar + dst_ty = elt_ty + + # Build IR + if mask is None: + ret = tl.tensor(builder.create_load(ptr.handle, cache, eviction, is_volatile), dst_ty) + else: + ret = tl.tensor( + builder.create_masked_load(ptr.handle, mask.handle, other.handle if other else None, cache, eviction, + is_volatile), dst_ty) + if is_bool: + ret = cast(ret, tl.int1, builder) + return ret + + +def load(ptr: tl.tensor, mask: Optional[tl.tensor], other: Optional[tl.tensor], boundary_check: Tuple, + padding_option: str, cache_modifier: str, eviction_policy: str, is_volatile: bool, + builder: ir.builder) -> tl.tensor: + # Cache, eviction and padding options + cache = _str_to_load_cache_modifier(cache_modifier) + eviction = _str_to_eviction_policy(eviction_policy) + padding = _str_to_padding_option(padding_option) + + if ptr.type.is_ptr() and ptr.type.element_ty.is_block(): + # Load by a block pointer: `pointer_type>` + return _load_block_pointer(ptr, mask, other, boundary_check, padding, cache, eviction, is_volatile, builder) + else: + # Load by a tensor of pointers or a pointer of scalar: `block_type>` or `pointer_type<>` + return _load_legacy(ptr, mask, other, boundary_check, padding, cache, eviction, is_volatile, builder) + + +def descriptor_load(desc_ptr: tl.tensor, offsets, cache_modifier: str, eviction_policy: str, type, + builder: ir.builder) -> tl.tensor: + offsets = _convert_to_ir_values(builder, offsets, require_i64=False) + x = builder.create_descriptor_load(desc_ptr.handle, offsets, type.to_ir(builder), + _str_to_load_cache_modifier(cache_modifier), + _str_to_eviction_policy(eviction_policy)) + return tl.tensor(x, type) + + +def descriptor_store(desc_ptr: tl.tensor, value: tl.tensor, offsets, builder: ir.builder) -> tl.tensor: + offsets = _convert_to_ir_values(builder, offsets, require_i64=False) + return tl.tensor(builder.create_descriptor_store(desc_ptr.handle, value.handle, offsets), tl.void) + + +def tensormap_create( + desc_ptr: tl.tensor, + global_address: tl.tensor, + box_dim: List[tl.tensor], + global_dim: List[tl.tensor], + global_stride: List[tl.tensor], + element_stride: List[tl.tensor], + elem_type: int, + interleave_layout: int, + swizzle_mode: int, + fill_mode: int, + builder: ir.builder, +) -> tl.tensor: + assert not global_stride or global_stride[0].dtype == tl.int64 + return tl.tensor( + builder.create_tensormap_create( + desc_ptr.handle, + global_address.handle, + [x.handle for x in box_dim], + [x.handle for x in global_dim], + [x.handle for x in global_stride], + [x.handle for x in element_stride], + elem_type, + interleave_layout, + swizzle_mode, + fill_mode, + ), + tl.void, + ) + + +def tensormap_fenceproxy_acquire(desc_ptr: tl.tensor, builder: ir.builder) -> tl.tensor: + return tl.tensor(builder.create_tensormap_fenceproxy_acquire(desc_ptr.handle), tl.void) + + +def _store_block_pointer(ptr, val, mask, boundary_check, cache, eviction, builder): + # Store by a block pointer: `pointer_type>` + # Block pointers can not have the `mask` argument + if mask is not None: + raise ValueError("`mask` and `other` arguments cannot be specified for loading block pointers") + + # Check same shape and element type + block_shape = ptr.type.element_ty.get_block_shapes() + if not val.type.is_block(): + val = broadcast_impl_shape(val, block_shape, builder) + assert val.type.is_block(), "Value argument must be block type or a scalar" + assert block_shape == val.type.get_block_shapes( + ), f"Block shape({block_shape}) and value shape({val.type.get_block_shapes()}) mismatch" + assert ptr.type.element_ty.element_ty == val.type.element_ty, f"Block element type({ptr.type.element_ty.element_ty}) and value element type({val.type.element_ty}) mismatch" + + elt_ty = ptr.type.element_ty.element_ty + assert elt_ty != tl.int1, "`tl.int1` should be rewrited in `tl.make_block_ptr`" + + # Check `boundary_check` argument + boundary_check = _canonicalize_boundary_check(boundary_check, block_shape) + + # Cast to target data type + val = cast(val, elt_ty, builder) + + # Build IR + return tl.tensor(builder.create_tensor_pointer_store(ptr.handle, val.handle, boundary_check, cache, eviction), + tl.void) + + +def _store_legacy(ptr, val, mask, boundary_check, cache, eviction, builder): + # Store by a tensor of pointers or a pointer of scalar: `block_type>` or `pointer_type<>` + if not ptr.type.scalar.is_ptr(): + raise ValueError(f"Unsupported ptr type {ptr.type.__repr__()} in `tl.store`") + + # Check `boundary_check` argument + if boundary_check: + raise ValueError("`boundary_check` argument is not supported for storing a tensor of pointers or storing a " + "scalar. Because the compiler does not know the boundary; please use block pointers " + "(defined by `make_block_ptr`) instead") + + # For a pointer of scalar, check the type of `val` and `mask` + if not ptr.type.is_block(): + if val.type.is_block(): + raise ValueError("Value argument cannot be block type if pointer argument is not a block") + if mask and mask.type.is_block(): + raise ValueError("Mask argument cannot be block type if pointer argument is not a block") + + # Make `mask` and `val` into the same shape as `ptr` + if ptr.type.is_block(): + val = broadcast_impl_shape(val, ptr.type.get_block_shapes(), builder) + if mask is not None: + mask = broadcast_impl_shape(mask, ptr.type.get_block_shapes(), builder) + + ptr_ty = ptr.type.scalar + elt_ty = ptr_ty.element_ty + + # Treat `pointer_type` as `pointer_type` + if elt_ty == tl.int1: + elt_ty = tl.int8 + ptr_ty = tl.pointer_type(elt_ty, ptr_ty.address_space) + ptr = cast(ptr, ptr_ty, builder) + + # Cast to target data type + val = cast(val, elt_ty, builder) + + # Build IR + if not mask: + return tl.tensor(builder.create_store(ptr.handle, val.handle, cache, eviction), tl.void) + if not mask.type.scalar.is_bool(): + raise ValueError("Mask must have boolean scalar type") + return tl.tensor(builder.create_masked_store(ptr.handle, val.handle, mask.handle, cache, eviction), tl.void) + + +def store(ptr: tl.tensor, val: tl.tensor, mask: Optional[tl.tensor], boundary_check, cache_modifier: str, + eviction_policy: str, builder: ir.builder) -> tl.tensor: + # Cache and eviction options + cache = _str_to_store_cache_modifier(cache_modifier) + eviction = _str_to_eviction_policy(eviction_policy) + + if ptr.type.is_const() or ptr.type.scalar.is_const(): + raise ValueError("Cannot store to a constant pointer") + + if ptr.type.is_ptr() and ptr.type.element_ty.is_block(): + # Store by a block pointer: `pointer_type>` + return _store_block_pointer(ptr, val, mask, boundary_check, cache, eviction, builder) + else: + # Store by a tensor of pointers or a pointer of scalar: `block_type>` or `pointer_type<>` + return _store_legacy(ptr, val, mask, boundary_check, cache, eviction, builder) + + +######### +# atomic +######### + + +def atomic_cas(ptr: tl.tensor, cmp: tl.tensor, val: tl.tensor, sem: str, scope: str, builder: ir.builder) -> tl.tensor: + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + element_ty = ptr.type.scalar.element_ty + if element_ty.primitive_bitwidth not in [16, 32, 64]: + raise ValueError("atomic_cas only supports elements with width {16, 32, 64}") + return tl.tensor(builder.create_atomic_cas(ptr.handle, cmp.handle, val.handle, sem, scope), val.type) + + +def atom_red_typechecking_impl(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, op: str, + builder: ir.builder) -> Tuple[tl.tensor, tl.tensor, tl.tensor]: + if not ptr.type.scalar.is_ptr(): + raise ValueError("Pointer argument of store instruction is " + ptr.type.__repr__()) + if ptr.type.is_const() or ptr.type.element_ty.is_const(): + raise ValueError("Cannot store to a constant pointer") + element_ty = ptr.type.scalar.element_ty + if element_ty is tl.float16 and op != 'add': + raise ValueError("atomic_" + op + " does not support fp16") + if element_ty in [tl.int1, tl.int8, tl.int16, tl.bfloat16]: + raise ValueError("atomic_" + op + " does not support " + str(element_ty)) + if ptr.type.is_block(): + if mask is not None: + mask = broadcast_impl_shape(mask, ptr.type.get_block_shapes(), builder) + if val is not None: + val = broadcast_impl_shape(val, ptr.type.get_block_shapes(), builder) + val = cast(val, ptr.type.scalar.element_ty, builder) + if not mask: + mask_ir = builder.get_int1(True) + mask_ty = tl.int1 + if ptr.type.is_block(): + mask_ir = builder.create_splat(mask_ir, ptr.type.get_block_shapes()) + mask_ty = tl.block_type(tl.int1, ptr.type.get_block_shapes()) + mask = tl.tensor(mask_ir, mask_ty) + return ptr, val, mask + + +def atomic_max(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, sem: str, scope: str, builder: ir.builder) -> tl.tensor: + ptr, val, mask = atom_red_typechecking_impl(ptr, val, mask, 'max', builder) + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + sca_ty = val.type.scalar + # direct call to atomic_max for integers + if sca_ty.is_int(): + if sca_ty.is_int_signed(): + return tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.MAX, ptr.handle, val.handle, mask.handle, sem, scope), val.type) + else: + return tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.UMAX, ptr.handle, val.handle, mask.handle, sem, scope), val.type) + # for float + # return atomic_smax(i_ptr, i_val) if val >= 0 + # return atomic_umin(i_ptr, i_val) if val < 0 + if sca_ty not in {tl.float32, tl.float64}: + raise TypeError(f"atomic_max not supported for dtype {sca_ty}") + + zero = full([], 0.0, sca_ty, builder) + + i_type = tl.int32 if sca_ty == tl.float32 else tl.int64 + i_val = bitcast(val, i_type, builder) + i_ptr = bitcast(ptr, tl.pointer_type(i_type, 1), builder) + ui_type = tl.uint32 if sca_ty == tl.float32 else tl.uint64 + ui_val = bitcast(val, ui_type, builder) + ui_ptr = bitcast(ptr, tl.pointer_type(ui_type, 1), builder) + pos = greater_equal(val, zero, builder) + neg = less_than(val, zero, builder) + pos_ret = tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.MAX, i_ptr.handle, i_val.handle, + and_(mask, pos, builder).handle, sem, scope), i_val.type) + neg_ret = tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.UMIN, ui_ptr.handle, ui_val.handle, + and_(mask, neg, builder).handle, sem, scope), ui_val.type) + ret = where(pos, pos_ret, neg_ret, builder) + return bitcast(ret, sca_ty, builder) + + +def atomic_min(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, sem: str, scope: str, builder: ir.builder) -> tl.tensor: + ptr, val, mask = atom_red_typechecking_impl(ptr, val, mask, 'min', builder) + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + sca_ty = val.type.scalar + # direct call to atomic_min for integers + if sca_ty.is_int(): + if sca_ty.is_int_signed(): + return tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.MIN, ptr.handle, val.handle, mask.handle, sem, scope), val.type) + else: + return tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.UMIN, ptr.handle, val.handle, mask.handle, sem, scope), val.type) + # for float + # return atomic_smin(i_ptr, i_val) if val >= 0 + # return atomic_umax(i_ptr, i_val) if val < 0 + if sca_ty not in {tl.float32, tl.float64}: + raise TypeError(f"atomic_min not supported for dtype {sca_ty}") + + zero = full([], 0.0, sca_ty, builder) + + i_type = tl.int32 if sca_ty == tl.float32 else tl.int64 + i_val = bitcast(val, i_type, builder) + i_ptr = bitcast(ptr, tl.pointer_type(i_type, 1), builder) + ui_type = tl.uint32 if sca_ty == tl.float32 else tl.uint64 + ui_val = bitcast(val, ui_type, builder) + ui_ptr = bitcast(ptr, tl.pointer_type(ui_type, 1), builder) + pos = greater_equal(val, zero, builder) + neg = less_than(val, zero, builder) + pos_ret = tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.MIN, i_ptr.handle, i_val.handle, + and_(mask, pos, builder).handle, sem, scope), i_val.type) + neg_ret = tl.tensor( + builder.create_atomic_rmw(ir.ATOMIC_OP.UMAX, ui_ptr.handle, ui_val.handle, + and_(mask, neg, builder).handle, sem, scope), ui_ptr.type) + ret = where(pos, pos_ret, neg_ret, builder) + return bitcast(ret, sca_ty, builder) + + +def atomic_add(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, sem: str, scope: str, builder: ir.builder) -> tl.tensor: + ptr, val, mask = atom_red_typechecking_impl(ptr, val, mask, 'add', builder) + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + sca_ty = val.type.scalar + op = ir.ATOMIC_OP.FADD if sca_ty.is_floating() else ir.ATOMIC_OP.ADD + return tl.tensor(builder.create_atomic_rmw(op, ptr.handle, val.handle, mask.handle, sem, scope), val.type) + + +def atomic_and(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, sem: str, scope: str, builder: ir.builder) -> tl.tensor: + ptr, val, mask = atom_red_typechecking_impl(ptr, val, mask, 'and', builder) + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + return tl.tensor(builder.create_atomic_rmw(ir.ATOMIC_OP.AND, ptr.handle, val.handle, mask.handle, sem, scope), + val.type) + + +def atomic_or(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, sem: str, scope: str, builder: ir.builder) -> tl.tensor: + ptr, val, mask = atom_red_typechecking_impl(ptr, val, mask, 'or', builder) + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + return tl.tensor(builder.create_atomic_rmw(ir.ATOMIC_OP.OR, ptr.handle, val.handle, mask.handle, sem, scope), + val.type) + + +def atomic_xor(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, sem: str, scope: str, builder: ir.builder) -> tl.tensor: + ptr, val, mask = atom_red_typechecking_impl(ptr, val, mask, 'xor', builder) + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + return tl.tensor(builder.create_atomic_rmw(ir.ATOMIC_OP.XOR, ptr.handle, val.handle, mask.handle, sem, scope), + val.type) + + +def atomic_xchg(ptr: tl.tensor, val: tl.tensor, mask: tl.tensor, sem: str, scope: str, + builder: ir.builder) -> tl.tensor: + ptr, val, mask = atom_red_typechecking_impl(ptr, val, mask, 'xchg', builder) + sem = _str_to_sem(sem) + scope = _str_to_scope(scope) + return tl.tensor(builder.create_atomic_rmw(ir.ATOMIC_OP.XCHG, ptr.handle, val.handle, mask.handle, sem, scope), + val.type) + + +# ===----------------------------------------------------------------------===// +# Linear Algebra +# ===----------------------------------------------------------------------===// + + +def _str_to_dot_input_precision(input_precision, builder): + assert input_precision.lower() in builder.options.allowed_dot_input_precisions, \ + f"input_precision must be one of {builder.options.allowed_dot_input_precisions}. Got {input_precision}" + input_precision = input_precision.upper() + if input_precision == "TF32X3": + input_precision = "TF32x3" + return getattr(ir.INPUT_PRECISION, input_precision) + + +def dot(lhs: tl.tensor, rhs: tl.tensor, acc: tl.tensor, input_precision: Optional[str], max_num_imprecise_acc: int, + out_dtype: tl.dtype, builder: ir.builder) -> tl.tensor: + assert lhs.type.is_block() and rhs.type.is_block() + + if lhs.dtype.is_fp8() and rhs.dtype.is_fp8(): + # All combinations of supported fp8 x fp8 are permitted + pass + else: + assert lhs.dtype in (tl.int8, tl.uint8, tl.float16, tl.bfloat16, + tl.float32), f"Unsupported lhs dtype {lhs.dtype}" + assert rhs.dtype in (tl.int8, tl.uint8, tl.float16, tl.bfloat16, + tl.float32), f"Unsupported rhs dtype {rhs.dtype}" + assert lhs.dtype == rhs.dtype, f"Both operands must be same dtype. Got {lhs.dtype} and {rhs.dtype}" + + if lhs.dtype.is_fp8e4b15() or rhs.dtype.is_fp8e4b15(): + lhs = cast(lhs, tl.float16, builder) + rhs = cast(rhs, tl.float16, builder) + + if input_precision is None: + input_precision = builder.options.default_dot_input_precision + + input_precision = _str_to_dot_input_precision(input_precision, builder) + + lhs_rank = len(lhs.shape) + rhs_rank = len(rhs.shape) + assert lhs_rank == rhs_rank == 2 or lhs_rank == rhs_rank == 3, f"Both inputs must be either 2D or 3D; (lhs: {lhs.shape} vs rhs: {rhs.shape})" + assert lhs.shape[-1].value == rhs.shape[ + -2].value, f"First input shape ({lhs.shape}) and second input shape {rhs.shape} are not compatible for matmul (second index of first shape ({lhs.shape[-1].value}) must be equal to first index of second shape ({rhs.shape[-2].value})" + assert builder.codegen_fns.get("min_dot_size") is not None, "target doesn't provide lower shape bounds for dot." + min_dot_size = builder.codegen_fns["min_dot_size"](lhs.type, rhs.type) + assert lhs.shape[-2].value >= min_dot_size[0] and lhs.shape[-1].value >= min_dot_size[2] \ + and rhs.shape[-1].value >= min_dot_size[1], \ + f"Input shapes should have M >= {min_dot_size[0]}, N >= {min_dot_size[1]} and K >= {min_dot_size[2]}" + if lhs.type.scalar.is_int(): + assert lhs.type.scalar == tl.int8, "only int8 supported!" + _0 = builder.get_int32(0) + ret_scalar_ty = tl.int32 + elif out_dtype.is_bf16(): + raise ValueError( + "out_dtype=bfloat16 is unsupported. Please use out_dtype=float32/float16 and cast with `.to(tl.bfloat16)`") + elif lhs.type.scalar.is_fp32() or lhs.type.scalar.is_bf16(): + _0 = builder.get_fp32(0) + ret_scalar_ty = tl.float32 + else: + _0 = builder.get_fp16(0) if out_dtype.is_fp16() else builder.get_fp32(0) + ret_scalar_ty = out_dtype + + M = lhs.type.shape[-2] + N = rhs.type.shape[-1] + K = lhs.type.shape[-1] + B = lhs.type.shape[0] if lhs_rank == 3 else None + ret_ty = tl.block_type(ret_scalar_ty, [B, M, N] if B else [M, N]) + if acc is None: + acc_handle = builder.create_splat(_0, [B, M, N] if B else [M, N]) + else: + acc_handle = acc.handle + assert acc.type == ret_ty + + # max_num_imprecise_acc only applies to fp8 -> fp32 dot on sm_90 + if max_num_imprecise_acc is None: + if lhs.dtype.is_fp8() and rhs.dtype.is_fp8(): + max_num_imprecise_acc = builder.options.max_num_imprecise_acc_default + else: + max_num_imprecise_acc = 0 + else: + if lhs.dtype.is_fp8() and rhs.dtype.is_fp8() and max_num_imprecise_acc > K: + raise ValueError(f"max_num_imprecise_acc ({max_num_imprecise_acc}) must be <= K ({K})") + + return tl.tensor(builder.create_dot(lhs.handle, rhs.handle, acc_handle, input_precision, max_num_imprecise_acc), + ret_ty) + + +def _str_to_fp_type(float_format: Optional[str]): + if float_format == 'e4m3': + return ir.F8F6F4TY.E4M3 + if float_format == 'e5m2': + return ir.F8F6F4TY.E5M2 + if float_format == 'e2m3': + return ir.F8F6F4TY.E2M3 + if float_format == 'e3m2': + return ir.F8F6F4TY.E3M2 + if float_format == 'e2m1': + return ir.F8F6F4TY.E2M1 + raise ValueError(f"Invalid float format: {float_format}.") + + +def dot_scaled(lhs: tl.tensor, lhs_scale: tl.tensor, lhs_format, rhs: tl.tensor, rhs_scale: Optional[tl.tensor], + rhs_format, acc: tl.tensor | None, out_dtype: tl.dtype, builder: ir.builder) -> tl.tensor: + assert lhs.type.is_block() and rhs.type.is_block() + #TODO: validate types. + lhs_rank = len(lhs.shape) + rhs_rank = len(rhs.shape) + assert lhs_rank == rhs_rank == 2 or lhs_rank == rhs_rank == 3, f"Both inputs must be either 2D or 3D; (lhs: {lhs.shape} vs rhs: {rhs.shape})" + lhs_format_enum = _str_to_fp_type(lhs_format) + rhs_format_enum = _str_to_fp_type(rhs_format) + assert lhs_format in ("e2m1", "e4m3", "e5m2"), f"NYI: lhs_format {lhs_format}" + assert rhs_format in ("e4m3", "e5m2"), f"NYI: rhs_format {rhs_format}" + rhs_scale_is_none = isinstance(rhs_scale, tl.constexpr) and rhs_scale.value is None + assert rhs_scale_is_none, "NYI: rhs_scale not supported" + + M = lhs.type.shape[-2] + K, N = rhs.type.shape[-2:] + PACKED = 2 if lhs_format == "e2m1" else 1 + assert K == PACKED * lhs.type.shape[ + -1], f"Reduction dimension should pack the same number of elements; (lhs: {lhs.shape} vs rhs: {rhs.shape})" + assert K >= 64, f"scaled_dot NYI for K < 64. Got {K=}" + B = lhs.type.shape[0] if lhs_rank == 3 else None + + ret_ty = tl.block_type(out_dtype, [B, M, N] if B else [M, N]) + _0 = builder.get_fp32(0) + if acc is None: + acc_handle = builder.create_splat(_0, [B, M, N] if B else [M, N]) + else: + acc_handle = acc.handle + assert acc.type == ret_ty + rhs_scale_handle = None if rhs_scale_is_none else rhs_scale.handle + return tl.tensor( + builder.create_dot_scaled(lhs.handle, lhs_scale.handle, lhs_format_enum, rhs.handle, rhs_scale_handle, + rhs_format_enum, acc_handle), ret_ty) + + +# ===----------------------------------------------------------------------===// +# Indexing +# ===----------------------------------------------------------------------===// + + +def where(condition: tl.tensor, x: tl.tensor, y: tl.tensor, builder: ir.builder) -> tl.tensor: + if condition.dtype != tl.int1: + warnings.warn( + f"tl.where with a non-boolean condition is deprecated and will error out in a future triton release. Got {condition.dtype}" + ) + condition = cast(condition, tl.int1, builder) + x, y = binary_op_type_checking_impl(x, y, builder, True, True) + # x, y are broadcasted + if condition.type.is_block(): + condition, x = broadcast_impl_value(condition, x, builder) + x, y = broadcast_impl_value(x, y, builder) + else: + condition, _ = broadcast_impl_value(condition, x, builder) + ret_ty = x.type + return tl.tensor(builder.create_select(condition.handle, x.handle, y.handle), ret_ty) + + +# ===----------------------------------------------------------------------===// +# Reduction +# ===----------------------------------------------------------------------=== + + +def wrap_tensor(x, scalar_ty, ret_shape): + if ret_shape: + res_ty = tl.block_type(scalar_ty, ret_shape) + else: + # 0d-tensor -> scalar + res_ty = scalar_ty + return tl.tensor(x, res_ty) + + +def reduction(inputs: Sequence[tl.tensor], axis: int, region_builder_fn, builder: ir.builder) -> Tuple[tl.tensor, ...]: + if axis is None: + inputs = tuple(reshape(t, [t.numel.value], can_reorder=True, builder=builder) for t in inputs) + axis = 0 + # get result shape + shape = inputs[0].type.shape + rank = len(shape) + assert axis < rank, f"reduction axis must be < inputs rank ({rank})" + ret_shape = [s for i, s in enumerate(shape) if i != axis] + assert all(t.type.shape == shape for t in inputs), "all reduction inputs must have the same shape" + + reduce_op = builder.create_reduce([t.handle for t in inputs], axis) + region_builder_fn(reduce_op) + reduce_op.verify() + + return tuple(wrap_tensor(reduce_op.get_result(i), inputs[i].type.scalar, ret_shape) for i in range(len(inputs))) + + +# ===----------------------------------------------------------------------=== +# Associative Scan +# ===----------------------------------------------------------------------=== + + +def associative_scan(inputs: Sequence[tl.tensor], axis: int, region_builder_fn, reverse: bool, + builder: ir.builder) -> Tuple[tl.tensor, ...]: + shape = inputs[0].type.shape + rank = len(shape) + + assert -rank <= axis < rank, f"scan axis {axis} must be < inputs rank ({rank})" + + if axis < 0: + axis += rank + + for t in inputs: + assert t.type.shape == shape, "all scan inputs must have the same shape" + + scan_op = builder.create_scan([t.handle for t in inputs], axis, reverse) + region_builder_fn(scan_op) + scan_op.verify() + + return tuple(wrap_tensor(scan_op.get_result(i), inputs[i].type.scalar, shape) for i in range(len(inputs))) + + +# ===----------------------------------------------------------------------=== +# Histogram +# ===----------------------------------------------------------------------=== + + +def histogram(input: tl.tensor, num_bins: int, builder: ir.builder) -> tl.tensor: + assert len(input.shape) == 1, "histogram only supports 1D input" + assert input.dtype.is_int(), "histogram only supports integer input" + return tl.tensor(builder.create_histogram(input.handle, num_bins), tl.block_type(tl.int32, (num_bins, ))) + + +## + + +def multiple_of(x: tl.tensor, values: List[int]) -> tl.tensor: + if max(1, len(x.shape)) != len(values): + raise ValueError("Shape of input to multiple_of does not match the length of values") + x.handle.set_attr("tt.divisibility", ir.make_attr(values, x.handle.get_context())) + return x + + +def max_contiguous(x: tl.tensor, values: List[int]) -> tl.tensor: + if len(x.shape) != len(values): + raise ValueError("Shape of input to max_contiguous does not match the length of values") + x.handle.set_attr("tt.contiguity", ir.make_attr(values, x.handle.get_context())) + return x + + +def max_constancy(x: tl.tensor, values: List[int]) -> tl.tensor: + if len(x.shape) != len(values): + raise ValueError("Shape of input to max_constancy does not match the length of values") + x.handle.set_attr("tt.constancy", ir.make_attr(values, x.handle.get_context())) + return x + + +def debug_barrier(builder: ir.builder) -> tl.tensor: + return tl.tensor(builder.create_barrier(), tl.void) + + +def device_print(prefix: str, args: List[tl.tensor], hex: bool, builder: ir.builder) -> tl.tensor: + # It makes sense visually for prefix to end in ": "; make it so. Also, + # non-empty prefixes should start with " ". + if not prefix.endswith(" ") and args: + prefix += " " + if not prefix.endswith(": ") and args: + prefix = prefix[:-1] + ": " + if len(prefix) > 2 and not prefix.startswith(" "): + prefix = " " + prefix + + new_args = [arg.handle for arg in args] + is_signed = [arg.dtype in (tl.int1, tl.int8, tl.int16, tl.int32, tl.int64) for arg in args] + return tl.tensor(builder.create_print(prefix, hex, new_args, is_signed), tl.void) + + +def device_assert(cond: tl.tensor, msg: str, builder: ir.builder) -> tl.tensor: + if not builder.options.debug: + return + return tl.tensor(builder.create_assert(cond.handle, msg), tl.void) + + +def assume(cond, builder: ir.builder) -> tl.tensor: + return tl.tensor(builder.create_assume(cond.handle), tl.void) + + +def _convert_elem_to_ir_value(builder, elem, require_i64): + if isinstance(elem, int): + elem = tl.constexpr(elem) + if isinstance(elem, tl.constexpr): + if require_i64: + assert -2**63 <= elem.value < 2**63, f"Block pointers only support 64 bit `shape/strides`, " \ + f"got a value {elem.value} which is out of the range" + return builder.get_int64(elem.value) + else: + assert -2**31 <= elem.value < 2**31, f"Block pointers only support 32 bit `offsets/block_shape`, " \ + f"got a value {elem.value} which is out of the range" + return builder.get_int32(elem.value) + elif isinstance(elem, tl.tensor): + assert elem.numel.value == 1, "Expected a scalar in shape/strides/offsets" + assert elem.dtype.is_int(), "Expected an integer scalar type in shape/strides/offsets" + if elem.dtype != tl.int64 and require_i64: + return builder.create_int_cast(elem.handle, builder.get_int64_ty(), elem.dtype.is_int_signed()) + elif elem.dtype != tl.int32 and not require_i64: + assert False, "Block pointers only support 32 bit `offsets/block_shape`, " \ + "add a `.to(tl.int32)` or use regular indexing for 64 bit support" + return elem.handle + assert False, f"Unsupported element type in shape/strides/offsets: {type(elem)}" + + +def _convert_to_ir_values(builder, list_like, require_i64=True): + if hasattr(list_like, "__iter__"): + return [_convert_elem_to_ir_value(builder, elem, require_i64) for elem in list_like] + return [_convert_elem_to_ir_value(builder, list_like, require_i64)] + + +def make_block_ptr(base: tl.tensor, shape, strides, offsets, block_shape, order, builder: ir.builder) -> tl.tensor: + # Convert dynamic arguments to IR values + # NOTES(Chenggang): current `shape/strides` are `int64_t`, while `offsets/block_shape` are `int32_t` + shape = _convert_to_ir_values(builder, shape) + strides = _convert_to_ir_values(builder, strides) + offsets = _convert_to_ir_values(builder, offsets, require_i64=False) + + # Check `base` type + if not base.type.is_ptr() or base.type.element_ty.is_block(): + raise ValueError("Expected `base` to be a pointer type (but not a block pointer type or others)") + + # Treat `pointer_type` as `pointer_type` + if base.type.element_ty == tl.int1: + base = cast(base, tl.pointer_type(tl.int8, base.type.address_space), builder) + + # Check whether `block_shape` is static + if not hasattr(block_shape, "__iter__"): + block_shape = [block_shape] + block_shape = [elem.value if isinstance(elem, tl.constexpr) else elem for elem in block_shape] + assert all(isinstance(elem, int) and -2**31 <= elem < 2**31 for elem in block_shape), \ + "Expected a list of constant integers (`int32_t` range) in `block_shape`" + + # Check `order` + if not hasattr(order, "__iter__"): + order = [order] + order = [elem.value if isinstance(elem, tl.constexpr) else elem for elem in order] + assert sorted(order) == list(range(len(order))), "Expected a permutation of (0, 1, ..., len(order)-1) in order" + + # Must have same length + assert all(len(block_shape) == len(list_like) for list_like in [shape, strides, offsets, order]), \ + "Expected shape/strides/offsets/block_shape to have the same length" + + # Build value, the type is: + # `pointer_type>` in Python + # `tt.ptr>` in MLIR + handle = builder.create_make_block_ptr(base.handle, shape, strides, offsets, block_shape, order) + return tl.tensor(handle, tl.pointer_type(tl.block_type(base.type.element_ty, block_shape))) + + +def advance(base: tl.tensor, offsets, builder: ir.builder) -> tl.tensor: + # Convert dynamic offsets to IR values + offsets = _convert_to_ir_values(builder, offsets, require_i64=False) + + # Advanced block pointer type is the same as before + return tl.tensor(builder.create_advance(base.handle, offsets), base.type) diff --git a/lib/python3.12/site-packages/triton/language/standard.py b/lib/python3.12/site-packages/triton/language/standard.py new file mode 100644 index 0000000000000000000000000000000000000000..eaaeffb6878b2ea94ec2c74d9cf9ddb72aca1f93 --- /dev/null +++ b/lib/python3.12/site-packages/triton/language/standard.py @@ -0,0 +1,452 @@ +from __future__ import annotations + +from ..runtime.jit import jit +from . import core +from . import math + +# constexpr utilities + + +def _log2(i: core.constexpr): + log2 = 0 + n = i.value + while n > 1: + n >>= 1 + log2 += 1 + return core.constexpr(log2) + + +def _is_power_of_two(i: core.constexpr): + n = i.value + return core.constexpr((n & (n - 1)) == 0 and n != 0) + + +# ----------------------- +# Standard library +# ----------------------- + + +@core._tensor_member_fn +@jit +def cdiv(x, div): + """ + Computes the ceiling division of :code:`x` by :code:`div` + + :param x: the input number + :type x: Block + :param div: the divisor + :type div: Block + """ + return (x + div - 1) // div + + +@core._tensor_member_fn +@jit +@math._add_math_1arg_docstr("sigmoid") +def sigmoid(x): + return 1 / (1 + math.exp(-x)) + + +@core._tensor_member_fn +@jit +@math._add_math_1arg_docstr("softmax") +def softmax(x, ieee_rounding=False): + z = x - max(x, 0) + num = math.exp(z) + den = sum(num, 0) + return math.fdiv(num, den, ieee_rounding) + + +@core._tensor_member_fn +@jit +def ravel(x): + """ + Returns a contiguous flattened view of :code:`x`. + + :param x: the input tensor + :type x: Block + """ + return core.reshape(x, [x.numel], can_reorder=True) + + +@jit +def swizzle2d(i, j, size_i, size_j, size_g): + """ + Transforms the indices of a row-major `size_i * size_j` matrix into + the indices of a column-major matrix for each group of `size_g` rows. + + For example, for :code:`size_i = size_j = 4` and :code:`size_g = 2`, it will + transform :: + + [[0 , 1 , 2 , 3 ], + [4 , 5 , 6 , 7 ], + [8 , 9 , 10, 11], + [12, 13, 14, 15]] + + into :: + + [[0, 2, 4 , 6 ], + [1, 3, 5 , 7 ], + [8, 10, 12, 14], + [9, 11, 13, 15]] + """ + # "unrolled index in array" + ij = i * size_j + j + # number of elements in `size_g` groups + # of `size_j` columns + size_gj = size_g * size_j + # index of the group in which (i,j) is + group_id = ij // size_gj + # row-index of the first element of this group + off_i = group_id * size_g + # last group may have fewer rows + size_g = core.minimum(size_i - off_i, size_g) + # linear index with respect to the first element in this group + ij = ij % size_gj + # new row and column indices + new_i = off_i + ij % size_g + new_j = ij // size_g + return new_i, new_j + + +@jit +def zeros(shape, dtype): + """ + Returns a tensor filled with the scalar value 0 for the given :code:`shape` and :code:`dtype`. + + :param shape: Shape of the new array, e.g., (8, 16) or (8, ) + :type shape: tuple of ints + :param dtype: Data-type of the new array, e.g., :code:`tl.float16` + :type dtype: DType + """ + return core.full(shape, 0, dtype) + + +@jit +def zeros_like(input): + """ + Returns a tensor of zeros with the same shape and type as a given tensor. + + :param input: input tensor + :type input: Tensor + """ + return zeros(input.shape, input.dtype) + + +# max and argmax + + +@jit +def _argmax_combine(value1, index1, value2, index2, tie_break_left): + if tie_break_left: + tie = value1 == value2 and index1 < index2 + else: + tie = False + gt = value1 > value2 or tie + v_ret = core.where(gt, value1, value2) + i_ret = core.where(gt, index1, index2) + return v_ret, i_ret + + +@jit +def _argmax_combine_tie_break_left(value1, index1, value2, index2): + return _argmax_combine(value1, index1, value2, index2, True) + + +@jit +def _argmax_combine_tie_break_fast(value1, index1, value2, index2): + return _argmax_combine(value1, index1, value2, index2, False) + + +@jit +def _elementwise_max(a, b): + return core.maximum(a, b) + + +@core._tensor_member_fn +@jit +@core._add_reduction_docstr("maximum", return_indices_arg="return_indices", + tie_break_arg="return_indices_tie_break_left") +def max(input, axis=None, return_indices=False, return_indices_tie_break_left=True, keep_dims=False): + input = core._promote_bfloat16_to_float32(input) + if return_indices: + if return_indices_tie_break_left: + return core._reduce_with_indices(input, axis, _argmax_combine_tie_break_left, keep_dims=keep_dims) + else: + return core._reduce_with_indices(input, axis, _argmax_combine_tie_break_fast, keep_dims=keep_dims) + else: + if core.constexpr(input.dtype.primitive_bitwidth) < core.constexpr(32): + if core.constexpr(input.dtype.is_floating()): + input = input.to(core.float32) + else: + assert input.dtype.is_int(), "Expecting input to be integer type" + input = input.to(core.int32) + return core.reduce(input, axis, _elementwise_max, keep_dims=keep_dims) + + +@core._tensor_member_fn +@jit +@core._add_reduction_docstr("maximum index", tie_break_arg="tie_break_left") +def argmax(input, axis, tie_break_left=True, keep_dims=False): + (_, ret) = max(input, axis, return_indices=True, return_indices_tie_break_left=tie_break_left, keep_dims=keep_dims) + return ret + + +# min and argmin + + +@jit +def _argmin_combine(value1, index1, value2, index2, tie_break_left): + if tie_break_left: + tie = value1 == value2 and index1 < index2 + else: + tie = False + lt = value1 < value2 or tie + value_ret = core.where(lt, value1, value2) + index_ret = core.where(lt, index1, index2) + return value_ret, index_ret + + +@jit +def _argmin_combine_tie_break_left(value1, index1, value2, index2): + return _argmin_combine(value1, index1, value2, index2, True) + + +@jit +def _argmin_combine_tie_break_fast(value1, index1, value2, index2): + return _argmin_combine(value1, index1, value2, index2, False) + + +@jit +def _elementwise_min(a, b): + return core.minimum(a, b) + + +@core._tensor_member_fn +@jit +@core._add_reduction_docstr("minimum", return_indices_arg="return_indices", + tie_break_arg="return_indices_tie_break_left") +def min(input, axis=None, return_indices=False, return_indices_tie_break_left=True, keep_dims=False): + input = core._promote_bfloat16_to_float32(input) + if return_indices: + if return_indices_tie_break_left: + return core._reduce_with_indices(input, axis, _argmin_combine_tie_break_left, keep_dims=keep_dims) + else: + return core._reduce_with_indices(input, axis, _argmin_combine_tie_break_fast, keep_dims=keep_dims) + else: + if core.constexpr(input.dtype.primitive_bitwidth) < 32: + if core.constexpr(input.dtype.is_floating()): + input = input.to(core.float32) + else: + assert input.dtype.is_int(), "Expecting input to be integer type" + input = input.to(core.int32) + return core.reduce(input, axis, _elementwise_min, keep_dims=keep_dims) + + +@core._tensor_member_fn +@jit +@core._add_reduction_docstr("minimum index", tie_break_arg="tie_break_left") +def argmin(input, axis, tie_break_left=True, keep_dims=False): + _, ret = min(input, axis, return_indices=True, return_indices_tie_break_left=tie_break_left, keep_dims=keep_dims) + return ret + + +@jit +def _sum_combine(a, b): + return a + b + + +# sum + + +@core._tensor_member_fn +@jit +@core._add_reduction_docstr("sum") +def sum(input, axis=None, keep_dims=False): + input = core._promote_bfloat16_to_float32(input) + return core.reduce(input, axis, _sum_combine, keep_dims=keep_dims) + + +@jit +def _xor_combine(a, b): + return a ^ b + + +# xor sum + + +@core._tensor_member_fn +@core.builtin +@core._add_reduction_docstr("xor sum") +def xor_sum(input, axis=None, keep_dims=False, _builder=None, _generator=None): + scalar_ty = input.type.scalar + if not scalar_ty.is_int(): + raise ValueError("xor_sum only supported for integers") + + input = core._promote_bfloat16_to_float32(input, _builder=_builder) + return core.reduce(input, axis, _xor_combine, keep_dims=keep_dims, _builder=_builder, _generator=_generator) + + +# cumsum + + +@core._tensor_member_fn +@jit +@core._add_scan_docstr("cumsum") +def cumsum(input, axis=0, reverse=False): + # todo rename this to a generic function name + input = core._promote_bfloat16_to_float32(input) + return core.associative_scan(input, axis, _sum_combine, reverse) + + +# cumprod + + +@jit +def _prod_combine(a, b): + return a * b + + +@core._tensor_member_fn +@jit +@core._add_scan_docstr("cumprod") +def cumprod(input, axis=0, reverse=False): + # todo rename this to a generic function name + input = core._promote_bfloat16_to_float32(input) + return core.associative_scan(input, axis, _prod_combine, reverse) + + +# sort + + +@jit +def _compare_and_swap(x, flip, i: core.constexpr, n_dims: core.constexpr): + n_outer: core.constexpr = x.numel >> n_dims + shape: core.constexpr = [n_outer * 2**i, 2, 2**(n_dims - i - 1)] + y = core.reshape(x, shape) + # slice left/right with 'stride' 2**(n_dims - i - 1) + mask = core.arange(0, 2)[None, :, None] + left = core.broadcast_to(sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype) + right = core.broadcast_to(sum(y * mask, 1)[:, None, :], shape).to(y.dtype) + left = core.reshape(left, x.shape) + right = core.reshape(right, x.shape) + # actual compare-and-swap + idtype = core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True) + ileft = left.to(idtype, bitcast=True) + iright = right.to(idtype, bitcast=True) + ix = x.to(idtype, bitcast=True) + ret = ix ^ core.where((left > right) != flip, ileft ^ iright, zeros_like(ix)) + return ret.to(x.dtype, bitcast=True) + + +@jit +def _bitonic_merge(x, stage: core.constexpr, order: core.constexpr, n_dims: core.constexpr): + ''' + order_type 0 == ascending + order_type 1 == descending + order_type 2 == alternating + ''' + n_outer: core.constexpr = x.numel >> n_dims + core.static_assert(stage <= n_dims) + # flip denotes whether to re-arrange sub-sequences of elements in ascending or + # descending order. + # if flip = 00000000... then all elements will be re-arranged ascendingly at this stage + # if flip = 00110011... then all the elements will be re-arranged alternatingly (with + # a stride of 2) at this stage + if order == 2: + shape: core.constexpr = [n_outer * 2**(n_dims - 1 - stage), 2, 2**stage] + flip = core.reshape(core.broadcast_to(core.arange(0, 2)[None, :, None], shape), x.shape) + else: + flip = order + # perform `stage` rounds of `compare-and-swap` + for i in core.static_range(stage): + x = _compare_and_swap(x, flip, i + (n_dims - stage), n_dims) + return x + + +@core._tensor_member_fn +@jit +def sort(x, dim: core.constexpr = None, descending: core.constexpr = core.CONSTEXPR_0): + """ + Sorts a tensor along a specified dimension. + + :param x: The input tensor to be sorted. + :type x: Tensor + :param dim: The dimension along which to sort the tensor. If None, the tensor is sorted along the last dimension. Currently, only sorting along the last dimension is supported. + :type dim: int, optional + :param descending: If set to True, the tensor is sorted in descending order. If set to False, the tensor is sorted in ascending order. + :type descending: bool, optional + """ + # handle default dimension or check that it is the most minor dim + _dim: core.constexpr = len(x.shape) - 1 if dim is None else dim + core.static_assert(_dim == len(x.shape) - 1, "only minor dimension is currently supported") + # iteratively run bitonic merge-sort steps + n_dims: core.constexpr = _log2(x.shape[_dim]) + for i in core.static_range(1, n_dims + 1): + x = _bitonic_merge(x, i, 2 if i < n_dims else descending, n_dims) + return x + + +# flip + + +def _get_flip_dim(dim, shape): + dim = core._unwrap_if_constexpr(dim) + shape = core._unwrap_if_constexpr(shape) + if dim is None: + dim = len(shape) - 1 + assert dim == len(shape) - 1, "Currently only support flipping the last dimension" + return core.constexpr(dim) + + +@core._tensor_member_fn +@jit +def flip(x, dim=None): + """ + Flips a tensor `x` along the dimension `dim`. + + :param x: the first input tensor + :type x: Block + :param dim: the dimension to flip along (currently only final dimension supported) + :type dim: int + """ + core.static_assert(_is_power_of_two(x.shape[_get_flip_dim(dim, x.shape)])) + core.static_assert(_is_power_of_two(x.numel)) + # # reshape the tensor to have all dimensions be 2. + # # TODO: We shouldn't have to change the dimensions not sorted. + steps: core.constexpr = _log2(x.numel) + start: core.constexpr = _log2(x.numel) - _log2(x.shape[_get_flip_dim(dim, x.shape)]) + y = core.reshape(x, [2] * steps) + y = core.expand_dims(y, start) + flip = (core.arange(0, 2)[:, None] == 1 - core.arange(0, 2)) + for i in core.static_range(start, steps): + flip2 = flip + for j in core.static_range(0, steps + 1): + if j != i and j != i + 1: + flip2 = core.expand_dims(flip2, j) + y = sum(y * flip2, i + 1, keep_dims=True) + x = core.reshape(y, x.shape) + return x + + +@jit +def interleave(a, b): + """ + Interleaves the values of two tensors along their last dimension. The two tensors must have the same shape. + Equivalent to `tl.join(a, b).reshape(a.shape[:-1] + [2 * a.shape[-1]])` + + :param a: The first input tensor. + :type a: Tensor + :param b: The second input tensor. + :type b: Tensor + """ + c = core.join(a, b) + + if len(c.shape) == 1: + # We must have interleaved two scalars. + return c + else: + # This `else` is necessary because Triton's AST parser doesn't + # understand that if we take the `if` above we definitely don't run this + # `else`. + return core.reshape(c, c.shape[:-2] + [2 * c.shape[-2]]) diff --git a/lib/python3.12/site-packages/triton/profiler/__init__.py b/lib/python3.12/site-packages/triton/profiler/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0add689155c3dc28595e6ce1aeead48e133276dc --- /dev/null +++ b/lib/python3.12/site-packages/triton/profiler/__init__.py @@ -0,0 +1,10 @@ +# flake8: noqa +from .scope import scope, enter_scope, exit_scope +from .profile import ( + start, + activate, + deactivate, + finalize, + profile, + DEFAULT_PROFILE_NAME, +) diff --git a/lib/python3.12/site-packages/triton/profiler/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/triton/profiler/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..83ec696c60dc429cae65997b3c7112f3fcf82b34 Binary files /dev/null and b/lib/python3.12/site-packages/triton/profiler/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/profiler/__pycache__/flags.cpython-312.pyc b/lib/python3.12/site-packages/triton/profiler/__pycache__/flags.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d97cc2f22e24d4ea0c94706f97cdf4b0b1cee3aa Binary files /dev/null and b/lib/python3.12/site-packages/triton/profiler/__pycache__/flags.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/profiler/__pycache__/hook.cpython-312.pyc b/lib/python3.12/site-packages/triton/profiler/__pycache__/hook.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ebff60dfb9596d290931ae6470363edafdc97e14 Binary files /dev/null and b/lib/python3.12/site-packages/triton/profiler/__pycache__/hook.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/profiler/__pycache__/profile.cpython-312.pyc b/lib/python3.12/site-packages/triton/profiler/__pycache__/profile.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d3675275e26379a5772c1365e2bf24f96547e888 Binary files /dev/null and b/lib/python3.12/site-packages/triton/profiler/__pycache__/profile.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/profiler/__pycache__/proton.cpython-312.pyc b/lib/python3.12/site-packages/triton/profiler/__pycache__/proton.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..d9ac957a7ccd4f3c2071cff26d305d6e3f9dd157 Binary files /dev/null and b/lib/python3.12/site-packages/triton/profiler/__pycache__/proton.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/profiler/__pycache__/scope.cpython-312.pyc b/lib/python3.12/site-packages/triton/profiler/__pycache__/scope.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..3c09b6135202d615ee28de9f88fa3980c70e7be6 Binary files /dev/null and b/lib/python3.12/site-packages/triton/profiler/__pycache__/scope.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/profiler/__pycache__/viewer.cpython-312.pyc b/lib/python3.12/site-packages/triton/profiler/__pycache__/viewer.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f5ee8fece9d3707643fadab36ec7d624371fd4d2 Binary files /dev/null and b/lib/python3.12/site-packages/triton/profiler/__pycache__/viewer.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/profiler/flags.py b/lib/python3.12/site-packages/triton/profiler/flags.py new file mode 100644 index 0000000000000000000000000000000000000000..37c75b2433efb08f1c3d8a60a661da0e65fee763 --- /dev/null +++ b/lib/python3.12/site-packages/triton/profiler/flags.py @@ -0,0 +1,33 @@ +""" +This file contains the global flags used in the proton package. +""" + +# Whether to enable profiling. Default is False. +profiling_on = False +# Whether the script is run from the command line. Default is False. +command_line = False + + +def set_profiling_on(): + global profiling_on + profiling_on = True + + +def set_profiling_off(): + global profiling_on + profiling_on = False + + +def get_profiling_on(): + global profiling_on + return profiling_on + + +def set_command_line(): + global command_line + command_line = True + + +def is_command_line(): + global command_line + return command_line diff --git a/lib/python3.12/site-packages/triton/profiler/hook.py b/lib/python3.12/site-packages/triton/profiler/hook.py new file mode 100644 index 0000000000000000000000000000000000000000..94f94bee0b562c1c4522b5e3636cdcc4c49c34a3 --- /dev/null +++ b/lib/python3.12/site-packages/triton/profiler/hook.py @@ -0,0 +1,33 @@ +from .scope import enter_scope, exit_scope +from triton.compiler import CompiledKernel, LazyDict + +COMPUTE_METADATA_SCOPE_NAME = "__proton_launch_metadata" + + +class TritonHook: + flops_width = [8, 16, 32, 64] + metrics = [f"flops{width}" for width in flops_width] + ["bytes"] + ["flops"] + + @staticmethod + def enter(lazy_dict: LazyDict) -> None: + enter_scope(COMPUTE_METADATA_SCOPE_NAME) + metadata = lazy_dict.get() + exit_scope() + fn_metrics = {k: metadata[k] for k in TritonHook.metrics if k in metadata} + enter_scope(metadata["name"], triton_op=True, metrics=fn_metrics) + + @staticmethod + def exit(lazy_dict: LazyDict) -> None: + exit_scope(triton_op=True) + + +def register_triton_hook() -> None: + if CompiledKernel.launch_enter_hook is None: + CompiledKernel.launch_enter_hook = TritonHook.enter + CompiledKernel.launch_exit_hook = TritonHook.exit + + +def unregister_triton_hook() -> None: + if CompiledKernel.launch_enter_hook == TritonHook.enter: + CompiledKernel.launch_enter_hook = None + CompiledKernel.launch_exit_hook = None diff --git a/lib/python3.12/site-packages/triton/profiler/profile.py b/lib/python3.12/site-packages/triton/profiler/profile.py new file mode 100644 index 0000000000000000000000000000000000000000..808a1742a5b80493826f0bd2d5f2456425b779bf --- /dev/null +++ b/lib/python3.12/site-packages/triton/profiler/profile.py @@ -0,0 +1,205 @@ +import functools +import triton +import os + +from triton._C.libproton import proton as libproton +from .hook import register_triton_hook, unregister_triton_hook +from .flags import set_profiling_off, set_profiling_on, is_command_line +from typing import Optional + +DEFAULT_PROFILE_NAME = "proton" + + +def _select_backend() -> str: + backend = triton.runtime.driver.active.get_current_target().backend + if backend == "cuda": + return "cupti" + elif backend == "hip": + return "roctracer" + else: + raise ValueError("No backend is available for the current target.") + + +def _check_env(backend: str) -> None: + if backend == "roctracer": + hip_device_envs = ["HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"] + for env in hip_device_envs: + if os.getenv(env, None) is not None: + raise ValueError( + f"Proton does not work when the environment variable {env} is set on AMD GPUs. Please unset it and use `ROCR_VISIBLE_DEVICES` instead" + ) + + +def start( + name: Optional[str] = None, + *, + context: Optional[str] = "shadow", + data: Optional[str] = "tree", + backend: Optional[str] = None, + hook: Optional[str] = None, +): + """ + Start profiling with the given name and backend. + + Usage: + + ```python + proton.start("my_profile") + # do something + proton.finalize() + ``` + + Args: + name (str, optional): The name (with path) of the profiling session. + If not provided, the default name is "~/proton.hatchet". + backend (str, optional): The backend to use for profiling. + Available options are [None, "cupti", "cupti_pcsampling", "roctracer"]. + Defaults to None, which automatically selects the backend matching the current active runtime. + context (str, optional): The context to use for profiling. + Available options are ["shadow", "python"]. + Defaults to "shadow". + data (str, optional): The data structure to use for profiling. + Available options are ["tree"]. + Defaults to "tree". + hook (str, optional): The hook to use for profiling. + Available options are [None, "triton"]. + Defaults to None. + Returns: + session (int): The session ID of the profiling session. + """ + if is_command_line(): + # Ignore the start() call if the script is run from the command line. + return + + if name is None: + name = DEFAULT_PROFILE_NAME + + if backend is None: + backend = _select_backend() + + _check_env(backend) + + set_profiling_on() + if hook and hook == "triton": + register_triton_hook() + return libproton.start(name, context, data, backend) + + +def activate(session: Optional[int] = 0) -> None: + """ + Activate the specified session. + The profiling session will be active and data will be recorded. + + Args: + session (int): The session ID of the profiling session. Defaults to 0 (the first session started.) + + Returns: + None + """ + if is_command_line() and session != 0: + raise ValueError("Only one session can be activated when running from the command line.") + libproton.activate(session) + + +def deactivate(session: Optional[int] = 0) -> None: + """ + Stop the specified session. + The profiling session's data will still be in the memory, but no more data will be recorded. + + Args: + session (int): The session ID of the profiling session. Defaults to 0 (the first session started.) + + Returns: + None + """ + if is_command_line() and session != 0: + raise ValueError("Only one session can be deactivated when running from the command line.") + libproton.deactivate(session) + + +def finalize(session: Optional[int] = None, output_format: str = "hatchet") -> None: + """ + Finalizes a profiling session. + Flush and write the profiling data to the file specified by the session name. + + Args: + session (int, optional): The session ID to finalize. If None, all sessions are finalized. Defaults to None. + output_format (str, optional): The output format for the profiling results. + Aavailable options are ["hatchet"]. + + Returns: + None + """ + if session is None: + set_profiling_off() + libproton.finalize_all(output_format) + unregister_triton_hook() + else: + if is_command_line() and session != 0: + raise ValueError("Only one session can be finalized when running from the command line.") + libproton.finalize(session, output_format) + + +def _profiling( + func, + name: Optional[str] = None, + context: Optional[str] = "shadow", + data: Optional[str] = "tree", + backend: Optional[str] = None, + hook: Optional[str] = None, +): + """ + Context manager for profiling. Internally use only. + + Args: + See start() for the arguments. + + Returns: + wrapper (function): The wrapped function. + """ + + @functools.wraps(func) + def wrapper(*args, **kwargs): + session = start(name, context=context, data=data, backend=backend, hook=hook) + ret = func(*args, **kwargs) + deactivate(session) + return ret + + return wrapper + + +def profile( + func=None, + *, + name: Optional[str] = None, + context: Optional[str] = "shadow", + data: Optional[str] = "tree", + backend: Optional[str] = None, + hook: Optional[str] = None, +): + """ + Decorator for profiling. + + Usage: + + ```python + @proton.profile + def foo(): + pass + ``` + + Args: + See start() for the arguments. + + Returns: + decorator (function): The decorator function. + """ + if func is None: + # It's being used with parentheses, so return a decorator + def decorator(f): + return _profiling(f, name=name, context=context, data=data, backend=backend, hook=hook) + + return decorator + else: + # It's being used without parentheses, so apply the decorator directly + return _profiling(func, name=name, context=context, data=data, backend=backend, hook=hook) diff --git a/lib/python3.12/site-packages/triton/profiler/proton.py b/lib/python3.12/site-packages/triton/profiler/proton.py new file mode 100644 index 0000000000000000000000000000000000000000..cbb7a0b6f90dc101a2214d8827ea977f2e95a280 --- /dev/null +++ b/lib/python3.12/site-packages/triton/profiler/proton.py @@ -0,0 +1,82 @@ +import argparse +import sys +import os +from .profile import start, finalize, _select_backend +from .flags import set_command_line + + +def parse_arguments(): + parser = argparse.ArgumentParser( + description="The proton command utility for profiling scripts and pytest tests.", usage=""" + proton [options] script.py [script_args] [script_options] + proton [options] pytest [pytest_args] [script_options] + python -m triton.profiler.proton [options] script.py [script_args] [script_options] +""", formatter_class=argparse.RawTextHelpFormatter) + parser.add_argument("-n", "--name", type=str, help="Name of the profiling session") + parser.add_argument("-b", "--backend", type=str, help="Profiling backend", default=None, + choices=["cupti", "cupti_pcsampling", "roctracer"]) + parser.add_argument("-c", "--context", type=str, help="Profiling context", default="shadow", + choices=["shadow", "python"]) + parser.add_argument("-d", "--data", type=str, help="Profiling data", default="tree", choices=["tree"]) + parser.add_argument("-k", "--hook", type=str, help="Profiling hook", default=None, choices=[None, "triton"]) + parser.add_argument('target_args', nargs=argparse.REMAINDER, help='Subcommand and its arguments') + args = parser.parse_args() + return args, args.target_args + + +def is_pytest(script): + return os.path.basename(script) == 'pytest' + + +def execute_as_main(script, args): + script_path = os.path.abspath(script) + # Prepare a clean global environment + clean_globals = { + "__name__": "__main__", + "__file__": script_path, + "__builtins__": __builtins__, + sys.__name__: sys, + } + + original_argv = sys.argv + sys.argv = [script] + args + # Append the script's directory in case the script uses relative imports + sys.path.append(os.path.dirname(script_path)) + + # Execute in the isolated environment + try: + with open(script_path, 'rb') as file: + code = compile(file.read(), script_path, 'exec') + exec(code, clean_globals) + except Exception as e: + print(f"An error occurred while executing the script: {e}") + finally: + sys.argv = original_argv + + +def run_profiling(args, target_args): + backend = args.backend if args.backend else _select_backend() + + start(args.name, context=args.context, data=args.data, backend=backend, hook=args.hook) + + # Set the command line mode to avoid any `start` calls in the script. + set_command_line() + + script = target_args[0] + script_args = target_args[1:] if len(target_args) > 1 else [] + if is_pytest(script): + import pytest + pytest.main(script_args) + else: + execute_as_main(script, script_args) + + finalize() + + +def main(): + args, target_args = parse_arguments() + run_profiling(args, target_args) + + +if __name__ == "__main__": + main() diff --git a/lib/python3.12/site-packages/triton/profiler/scope.py b/lib/python3.12/site-packages/triton/profiler/scope.py new file mode 100644 index 0000000000000000000000000000000000000000..26d946a8c155f253efb9406b31cdae07dc252f9e --- /dev/null +++ b/lib/python3.12/site-packages/triton/profiler/scope.py @@ -0,0 +1,105 @@ +import threading +from functools import wraps +from typing import Optional, Union + +from .flags import get_profiling_on +from triton._C.libproton import proton as libproton + +thread_local_scopes = threading.local() + +MetricValueType = Union[float, int] +PropertyValueType = Union[float, int, str] + + +class scope: + """ + A context manager and decorator for entering and exiting a scope. + + Usage: + context manager: + ```python + with proton.scope("test0", {metric_name: metric_value}): + foo[1,](x, y) + ``` + + decorator: + ```python + @proton.scope("test0", {metric_name: metric_value}) + def foo(x, y): + ... + ``` + + Args: + name (str): The name of the scope. + metrics (dict[str, float], optional): The metrics of the scope. Default is None. + """ + + def __init__(self, name: str, metrics: Optional[dict[str, MetricValueType]] = None, + properties: Optional[dict[str, PropertyValueType]] = None) -> None: + self.name = name + self.metrics = metrics + self.properties = properties + + def __enter__(self): + if not get_profiling_on(): + return self + self.id = libproton.record_scope() + libproton.enter_scope(self.id, self.name) + if self.metrics: + libproton.add_metrics(self.id, self.metrics) + if self.properties: + libproton.set_properties(self.id, self.properties) + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + if not get_profiling_on(): + return + libproton.exit_scope(self.id, self.name) + + def __call__(self, func): + + @wraps(func) + def wrapper(*args, **kwargs): + if get_profiling_on(): + id = libproton.record_scope() + libproton.enter_scope(id, self.name) + if self.metrics: + libproton.add_metrics(id, self.metrics) + if self.properties: + libproton.set_properties(id, self.properties) + ret = func(*args, **kwargs) + if get_profiling_on(): + libproton.exit_scope(id, self.name) + return ret + + return wrapper + + +def enter_scope(name: str, *, triton_op: bool = False, metrics: Optional[dict[str, MetricValueType]] = None, + properties: Optional[dict[str, PropertyValueType]] = None) -> int: + if not get_profiling_on(): + return -1 + id = libproton.record_scope() + if not hasattr(thread_local_scopes, "scopes"): + thread_local_scopes.scopes = [] + thread_local_scopes.scopes.append((id, name)) + if triton_op: + libproton.enter_op(id, name) + else: + libproton.enter_scope(id, name) + if metrics: + libproton.add_metrics(id, metrics) + if properties: + libproton.set_properties(id, properties) + return id + + +def exit_scope(triton_op: bool = False) -> int: + if not get_profiling_on(): + return -1 + id, name = thread_local_scopes.scopes.pop() + if triton_op: + libproton.exit_op(id, name) + else: + libproton.exit_scope(id, name) + return id diff --git a/lib/python3.12/site-packages/triton/profiler/viewer.py b/lib/python3.12/site-packages/triton/profiler/viewer.py new file mode 100644 index 0000000000000000000000000000000000000000..fe7c98807c57b20d929cb940e5f142535b664f26 --- /dev/null +++ b/lib/python3.12/site-packages/triton/profiler/viewer.py @@ -0,0 +1,321 @@ +import argparse +from collections import namedtuple +import json +import pandas as pd +try: + import hatchet as ht + from hatchet.query import NegationQuery +except ImportError: + raise ImportError("Failed to import hatchet. `pip install llnl-hatchet` to get the correct version.") +import numpy as np +from triton.profiler.hook import COMPUTE_METADATA_SCOPE_NAME, TritonHook + + +def match_available_metrics(metrics, raw_metrics): + ret = [] + if metrics: + for metric in metrics: + metric = metric.lower() + for raw_metric in raw_metrics: + raw_metric_no_unit = raw_metric.split("(")[0].strip().lower() + if metric in (raw_metric, raw_metric_no_unit): + ret.append(raw_metric + " (inc)") + break + else: + ret = [raw_metrics[0] + " (inc)"] + if len(ret) == 0: + raise RuntimeError(f"Metric {metric} is not found. Use the --list flag to list available metrics") + return ret + + +def get_raw_metrics(file): + database = json.load(file) + device_info = database.pop(1) + gf = ht.GraphFrame.from_literal(database) + return gf, gf.show_metric_columns(), device_info + + +def get_min_time_flops(df, device_info): + min_time_flops = pd.DataFrame(0.0, index=df.index, columns=["min_time"]) + for device_type in device_info: + for device_index in device_info[device_type]: + arch = device_info[device_type][device_index]["arch"] + num_sms = device_info[device_type][device_index]["num_sms"] + clock_rate = device_info[device_type][device_index]["clock_rate"] + for width in TritonHook.flops_width: + idx = df["device_id"] == device_index + device_frames = df[idx] + if f"flops{width}" not in device_frames.columns: + continue + max_flops = 0 + if device_type == "CUDA": + if arch == "80": + max_flops = 624e12 / (width / 8) + elif arch == "89": + # TODO(Keren): Implement fp16 acc-> 660.6 fp8 + max_flops = (330.3 * 1e12) / (width / 8) + elif arch == "90": + # 114 sms and 1755mhz is the base number of sms and clock rate of H100 pcie + max_flops = ((num_sms / 114 * clock_rate / (1755 * 1e3) * 1513) * 1e12) / (width / 8) + elif device_type == "HIP": + if arch == "gfx90a": + max_flops = 383e12 / (width / 8) + elif arch == "gfx941" or arch == "gfx942": + max_flops = 2614.9e12 / (width / 8) + else: + raise ValueError(f"Unsupported device type: {device_type}") + min_time_flops.loc[idx, "min_time"] += device_frames[f"flops{width}"].fillna(0) / max_flops + return min_time_flops + + +def get_min_time_bytes(df, device_info): + min_time_bytes = pd.DataFrame(0.0, index=df.index, columns=["min_time"]) + for device_type in device_info: + for device_index in device_info[device_type]: + idx = df["device_id"] == device_index + device_frames = df[idx] + memory_clock_rate = device_info[device_type][device_index]["memory_clock_rate"] # in khz + bus_width = device_info[device_type][device_index]["bus_width"] # in bits + peak_bandwidth = 2 * bus_width * memory_clock_rate * 1e3 / 8 + min_time_bytes.loc[idx, "min_time"] += device_frames["bytes"] / peak_bandwidth + return min_time_bytes + + +FactorDict = namedtuple("FactorDict", ["name", "factor"]) +time_factor_dict = FactorDict("time", {"time/s": 1, "time/ms": 1e-3, "time/us": 1e-6, "time/ns": 1e-9}) +avg_time_factor_dict = FactorDict("avg_time", {f"avg_{key}": value for key, value in time_factor_dict.factor.items()}) +bytes_factor_dict = FactorDict("bytes", {"byte/s": 1, "gbyte/s": 1e9, "tbyte/s": 1e12}) + +derivable_metrics = { + **{key: bytes_factor_dict + for key in bytes_factor_dict.factor.keys()}, +} + +# FLOPS have a specific width to their metric +default_flop_factor_dict = {f"flop/s": 1, f"gflop/s": 1e9, f"tflop/s": 1e12} +derivable_metrics.update( + {key: FactorDict("flops", default_flop_factor_dict) + for key in default_flop_factor_dict.keys()}) +for width in TritonHook.flops_width: + factor_name = f"flops{width}" + factor_dict = {f"flop{width}/s": 1, f"gflop{width}/s": 1e9, f"tflop{width}/s": 1e12} + derivable_metrics.update({key: FactorDict(factor_name, factor_dict) for key in factor_dict.keys()}) + + +def derive_metrics(gf, metrics, raw_metrics, device_info): + derived_metrics = [] + internal_frame_indices = gf.dataframe["device_id"].isna() + + def get_time_seconds(df): + time_metric_name = match_available_metrics([time_factor_dict.name], raw_metrics)[0] + time_unit = (time_factor_dict.name + "/" + time_metric_name.split("(")[1].split(")")[0]) + return df[time_metric_name] * time_factor_dict.factor[time_unit] + + for metric in metrics: + if metric == "util": # Tensor core only + min_time_bytes = get_min_time_bytes(gf.dataframe, device_info) + min_time_flops = get_min_time_flops(gf.dataframe, device_info) + time_sec = get_time_seconds(gf.dataframe) + gf.dataframe["util (inc)"] = min_time_flops["min_time"].combine(min_time_bytes["min_time"], max) / time_sec + gf.dataframe.loc[internal_frame_indices, "util (inc)"] = np.nan + derived_metrics.append("util (inc)") + elif metric in derivable_metrics: # flop/s, byte/s + derivable_metric = derivable_metrics[metric] + metric_name = derivable_metric.name + metric_factor_dict = derivable_metric.factor + matched_metric_name = match_available_metrics([metric_name], raw_metrics)[0] + gf.dataframe[f"{metric} (inc)"] = (gf.dataframe[matched_metric_name] / (get_time_seconds(gf.dataframe)) / + metric_factor_dict[metric]) + derived_metrics.append(f"{metric} (inc)") + elif metric in time_factor_dict.factor: + metric_time_unit = time_factor_dict.name + "/" + metric.split("/")[1] + gf.dataframe[f"{metric} (inc)"] = (get_time_seconds(gf.dataframe) / + time_factor_dict.factor[metric_time_unit]) + derived_metrics.append(f"{metric} (inc)") + elif metric in avg_time_factor_dict.factor: + metric_time_unit = avg_time_factor_dict.name + "/" + metric.split("/")[1] + gf.dataframe[f"{metric} (inc)"] = (get_time_seconds(gf.dataframe) / gf.dataframe['count'] / + avg_time_factor_dict.factor[metric_time_unit]) + gf.dataframe.loc[internal_frame_indices, f"{metric} (inc)"] = np.nan + derived_metrics.append(f"{metric} (inc)") + else: + metric_name_and_unit = metric.split("/") + metric_name = metric_name_and_unit[0] + if len(metric_name_and_unit) > 1: + metric_unit = metric_name_and_unit[1] + if metric_unit != "%": + raise ValueError(f"Unsupported unit {metric_unit}") + matched_metric_name = match_available_metrics([metric_name], raw_metrics)[0] + single_frame = gf.dataframe[matched_metric_name] + total = gf.dataframe[matched_metric_name].iloc[0] + gf.dataframe[f"{metric_name}/% (inc)"] = (single_frame / total) * 100.0 + derived_metrics.append(f"{metric_name}/% (inc)") + else: + matched_metric_name = match_available_metrics([metric_name], raw_metrics)[0] + derived_metrics.append(matched_metric_name) + return derived_metrics + + +def format_frames(gf, format): + if format == "file_function_line": + gf.dataframe["name"] = gf.dataframe["name"].apply(lambda x: x.split("/")[-1]) + elif format == "function_line": + gf.dataframe["name"] = gf.dataframe["name"].apply(lambda x: x.split(":")[-1]) + elif format == "file_function": + gf.dataframe["name"] = gf.dataframe["name"].apply(lambda x: x.split("/")[-1].split("@")[0]) + return gf + + +def filter_frames(gf, include=None, exclude=None, threshold=None, metric=None): + if include: + query = f""" +MATCH ("*")->(".", p)->("*") +WHERE p."name" =~ "{include}" +""" + gf = gf.filter(query, squash=True) + if exclude: + inclusion_query = f""" +MATCH (".", p)->("*") +WHERE p."name" =~ "{exclude}" +""" + query = NegationQuery(inclusion_query) + gf = gf.filter(query, squash=True) + # filter out metadata computation + query = [{"name": f"^(?!{COMPUTE_METADATA_SCOPE_NAME}).*"}] + gf = gf.filter(query, squash=True) + if threshold: + query = ["*", {metric: f">= {threshold}"}] + gf = gf.filter(query, squash=True) + return gf + + +def parse(metrics, filename, include=None, exclude=None, threshold=None, depth=100, format=None): + with open(filename, "r") as f: + gf, raw_metrics, device_info = get_raw_metrics(f) + gf = format_frames(gf, format) + assert len(raw_metrics) > 0, "No metrics found in the input file" + gf.update_inclusive_columns() + metrics = derive_metrics(gf, metrics, raw_metrics, device_info) + # TODO: generalize to support multiple metrics, not just the first one + gf = filter_frames(gf, include, exclude, threshold, metrics[0]) + print(gf.tree(metric_column=metrics, expand_name=True, depth=depth, render_header=False)) + emit_warnings(gf, metrics) + + +def emit_warnings(gf, metrics): + if "bytes (inc)" in metrics: + byte_values = gf.dataframe["bytes (inc)"].values + min_byte_value = np.nanmin(byte_values) + if min_byte_value < 0: + print("Warning: Negative byte values detected, this is usually the result of a datatype overflow\n") + + +def show_metrics(file_name): + with open(file_name, "r") as f: + _, raw_metrics, _ = get_raw_metrics(f) + print("Available metrics:") + if raw_metrics: + for raw_metric in raw_metrics: + raw_metric_no_unit = raw_metric.split("(")[0].strip().lower() + print(f"- {raw_metric_no_unit}") + + +def main(): + argparser = argparse.ArgumentParser( + description="Performance data viewer for proton profiles.", + formatter_class=argparse.RawTextHelpFormatter, + ) + argparser.add_argument( + "-l", + "--list", + action="store_true", + help="""List available metrics. Metric names are case insensitive and ignore units. +Derived metrics can be created when source metrics are available. +- time/s, time/ms, time/us, time/ns: time +- avg_time/s, avg_time/ms, avg_time/us, avg_time/ns: time / count +- flop[<8/16/32/64>]/s, gflop[<8/16/32/64>]/s, tflop[<8/16/32/64>]/s: flops / time +- byte/s, gbyte/s, tbyte/s: bytes / time +- util: max(sum(flops) / peak_flops_time, sum(bytes) / peak_bandwidth_time) +- /%%: frame(metric) / sum(metric). Only availble for inclusive metrics (e.g. time) +""", + ) + argparser.add_argument( + "-m", + "--metrics", + type=str, + default=None, + help="""At maximum two metrics can be specified, separated by comma. +There are two modes: +1) Choose the output metric to display. It's case insensitive and ignore units. +2) Derive a new metric from existing metrics. +""", + ) + argparser.add_argument( + "-i", + "--include", + type=str, + default=None, + help= + """Find frames that match the given regular expression and return all nodes in the paths that pass through the matching frames. +For example, the following command will display all paths that contain frames that contains "test": +``` +proton-viewer -i ".*test.*" path/to/file.json +``` +""", + ) + argparser.add_argument( + "-e", + "--exclude", + type=str, + default=None, + help="""Exclude frames that match the given regular expression and their children. +For example, the following command will exclude all paths that contain frames that contains "test": +``` +proton-viewer -e ".*test.*" path/to/file.json +``` +""", + ) + argparser.add_argument( + "-t", + "--threshold", + type=float, + default=None, + help= + "Exclude frames(kernels) whose metrics are below the given threshold. This filter only applies on the first metric.", + ) + argparser.add_argument( + "-d", + "--depth", + type=int, + default=100, + help="The depth of the tree to display", + ) + argparser.add_argument( + "-f", "--format", type=str, choices=["full", "file_function_line", "function_line", "file_function"], + default="full", help="""Formatting the frame name. +- full: include the path, file name, function name and line number. +- file_function_line: include the file name, function name and line number. +- function_line: include the function name and line number. +- file_function: include the file name and function name. +""") + + args, target_args = argparser.parse_known_args() + assert len(target_args) == 1, "Must specify a file to read" + + file_name = target_args[0] + metrics = args.metrics.split(",") if args.metrics else None + include = args.include + exclude = args.exclude + threshold = args.threshold + depth = args.depth + format = args.format + if include and exclude: + raise ValueError("Cannot specify both include and exclude") + if args.list: + show_metrics(file_name) + elif metrics: + parse(metrics, file_name, include, exclude, threshold, depth, format) + + +if __name__ == "__main__": + main() diff --git a/lib/python3.12/site-packages/triton/runtime/__init__.py b/lib/python3.12/site-packages/triton/runtime/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..0b3979d28d9a4359be5ceb3d2dea08f4d56899e6 --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/__init__.py @@ -0,0 +1,23 @@ +from .autotuner import (Autotuner, Config, Heuristics, autotune, heuristics) +from .cache import RedisRemoteCacheBackend, RemoteCacheBackend +from .driver import driver +from .jit import JITFunction, KernelInterface, MockTensor, TensorWrapper, reinterpret +from .errors import OutOfResources, InterpreterError + +__all__ = [ + "autotune", + "Autotuner", + "Config", + "driver", + "Heuristics", + "heuristics", + "InterpreterError", + "JITFunction", + "KernelInterface", + "MockTensor", + "OutOfResources", + "RedisRemoteCacheBackend", + "reinterpret", + "RemoteCacheBackend", + "TensorWrapper", +] diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a0d0b2a5a6590989c1f466dbdcea7f33ad0aa4dd Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/autotuner.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/autotuner.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..965d3ed49c845468d85e4bde78022f2ad63b569a Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/autotuner.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/build.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/build.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fc0c759612219a5cc61ff1a72d987f6aae97df81 Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/build.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/cache.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/cache.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..c44e705fdafddf999858c297c73e023b41f3b04f Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/cache.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/driver.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/driver.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..2e07477035a5a172f3b41656b320a0c41e71ffd3 Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/driver.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/errors.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/errors.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec27d56fadadf6947a8cf6c473a0a90279dfce06 Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/errors.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/interpreter.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/interpreter.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8f20928fee8c092f15403a4d007c5d07eb5586bd Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/interpreter.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/__pycache__/jit.cpython-312.pyc b/lib/python3.12/site-packages/triton/runtime/__pycache__/jit.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..788b643d15807cd9aa30598e68ac3d50a667275c Binary files /dev/null and b/lib/python3.12/site-packages/triton/runtime/__pycache__/jit.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/runtime/autotuner.py b/lib/python3.12/site-packages/triton/runtime/autotuner.py new file mode 100644 index 0000000000000000000000000000000000000000..9f494a06276a6a00a2f9ef3d499fe8247b38e478 --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/autotuner.py @@ -0,0 +1,408 @@ +from __future__ import annotations + +import builtins +import os +import time +import inspect +from typing import Dict + +from .jit import KernelInterface +from .errors import OutOfResources +from .driver import driver + + +class Autotuner(KernelInterface): + + def __init__( + self, + fn, + arg_names, + configs, + key, + reset_to_zero, + restore_value, + pre_hook=None, + post_hook=None, + prune_configs_by: Dict = None, + warmup=None, + rep=None, + use_cuda_graph=False, + do_bench=None, + ): + """ + :param prune_configs_by: a dict of functions that are used to prune configs, fields: + 'perf_model': performance model used to predicate running time with different configs, returns running time + 'top_k': number of configs to bench + 'prune_num_stages_by'(optional): a function used to prune num_stages. It takes configs:List[Config] as its input, and returns pruned configs. + """ + if not configs: + self.configs = [ + Config({}, num_warps=4, num_stages=2, num_ctas=1, num_buffers_warp_spec=0, num_consumer_groups=0, + reg_dec_producer=0, reg_inc_consumer=0) + ] + else: + self.configs = configs + self.keys = key + self.cache = {} + self.arg_names = arg_names + + # Reset to zero or restore values + self.reset_to_zero = [] + if reset_to_zero is not None: + self.reset_to_zero = list(reset_to_zero) + self.restore_value = [] + if restore_value is not None: + self.restore_value = list(restore_value) + + # Hook to reset or restore for required tensors + self.pre_hook = lambda kwargs, reset_only=False: 0 + self.post_hook = lambda kwargs, exception: 0 + self.user_defined_pre_hook = False + self.user_defined_post_hook = False + if pre_hook: + self.pre_hook = pre_hook + self.user_defined_pre_hook = True + elif (len(self.reset_to_zero) > 0 or len(self.restore_value) > 0): + + def _pre_hook(kwargs, reset_only=False): + for name in self.reset_to_zero: + kwargs[name].zero_() + if not reset_only: + self.restore_copies = {name: kwargs[name].clone() for name in self.restore_value} + + self.pre_hook = _pre_hook + + if post_hook: + self.post_hook = post_hook + self.user_defined_post_hook = True + elif len(self.restore_value) > 0: + + def _post_hook(kwargs, exception): + for name in self.restore_value: + kwargs[name].copy_(self.restore_copies[name]) + self.restore_copies = {} + + self.post_hook = _post_hook + + self.perf_model = None + self.configs_top_k = 1.0 + self.early_config_prune = None + if prune_configs_by: + self.perf_model = prune_configs_by.get("perf_model", self.perf_model) + self.configs_top_k = prune_configs_by.get("top_k", self.configs_top_k) + self.early_config_prune = prune_configs_by.get("early_config_prune", self.early_config_prune) + + self.fn = fn + self.base_fn = fn + while not inspect.isfunction(self.base_fn): + self.base_fn = self.base_fn.fn + + self.num_warmups = warmup + self.num_reps = rep + self.use_cuda_graph = use_cuda_graph + + # If we got explicitly called via the old interface, raise a warning + # and proceed with the old behavior. + if warmup is not None or rep is not None or use_cuda_graph: + import warnings + warnings.warn(("warmup, rep, and use_cuda_graph parameters are deprecated. See " + "https://github.com/triton-lang/triton/pull/4496 for details."), DeprecationWarning, + stacklevel=1) + if use_cuda_graph: + from ..testing import do_bench_cudagraph + self.do_bench = lambda kernel_call, quantiles: do_bench_cudagraph( + kernel_call, + rep=rep if rep is not None else 100, + quantiles=quantiles, + ) + return + + import triton.testing + self.do_bench = lambda kernel_call, quantiles: triton.testing.do_bench( + kernel_call, + warmup=warmup if warmup is not None else 25, + rep=rep if rep is not None else 100, + quantiles=quantiles, + ) + return + + if do_bench is None: + self.do_bench = driver.active.get_benchmarker() + else: + self.do_bench = do_bench + + def _bench(self, *args, config, **meta): + from ..compiler.errors import CompileTimeAssertionFailure + + # check for conflicts, i.e. meta-parameters both provided + # as kwargs and by the autotuner + conflicts = meta.keys() & config.kwargs.keys() + if conflicts: + raise ValueError(f"Conflicting meta-parameters: {', '.join(conflicts)}." + " Make sure that you don't re-define auto-tuned symbols.") + # augment meta-parameters with tunable ones + current = dict(meta, **config.all_kwargs()) + full_nargs = {**self.nargs, **current} + + def kernel_call(): + if config.pre_hook: + config.pre_hook(full_nargs) + self.pre_hook(full_nargs) + try: + self.fn.run( + *args, + **current, + ) + except Exception as e: + try: + self.post_hook(full_nargs, exception=e) + finally: + # Throw exception raised by `self.fn.run` + raise + + self.post_hook(full_nargs, exception=None) + + try: + return self.do_bench(kernel_call, quantiles=(0.5, 0.2, 0.8)) + except (OutOfResources, CompileTimeAssertionFailure): + return [float("inf"), float("inf"), float("inf")] + + def run(self, *args, **kwargs): + self.nargs = dict(zip(self.arg_names, args)) + used_cached_result = True + if len(self.configs) > 1: + all_args = {**self.nargs, **kwargs} + _args = {k: v for (k, v) in all_args.items() if k in self.arg_names} + key = [_args[key] for key in self.keys if key in _args] + for _, arg in _args.items(): + if hasattr(arg, "dtype"): + key.append(str(arg.dtype)) + key = tuple(key) + if key not in self.cache: + # prune configs + used_cached_result = False + pruned_configs = self.prune_configs(kwargs) + bench_start = time.time() + timings = {config: self._bench(*args, config=config, **kwargs) for config in pruned_configs} + bench_end = time.time() + self.bench_time = bench_end - bench_start + self.cache[key] = builtins.min(timings, key=timings.get) + full_nargs = {**self.nargs, **kwargs, **self.cache[key].all_kwargs()} + self.pre_hook(full_nargs, reset_only=True) + self.configs_timings = timings + config = self.cache[key] + else: + config = self.configs[0] + self.best_config = config + if os.getenv("TRITON_PRINT_AUTOTUNING", None) == "1" and not used_cached_result: + print(f"Triton autotuning for function {self.base_fn.__name__} finished after " + f"{self.bench_time:.2f}s; best config selected: {self.best_config};") + if config.pre_hook is not None: + full_nargs = {**self.nargs, **kwargs, **config.all_kwargs()} + config.pre_hook(full_nargs) + ret = self.fn.run( + *args, + **kwargs, + **config.all_kwargs(), + ) + self.nargs = None + return ret + + def prune_configs(self, kwargs): + pruned_configs = self.configs + if self.early_config_prune: + pruned_configs = self.early_config_prune(self.configs, self.nargs, **kwargs) + if self.perf_model: + top_k = self.configs_top_k + if isinstance(top_k, float) and top_k <= 1.0: + top_k = int(len(self.configs) * top_k) + if len(pruned_configs) > top_k: + est_timing = { + config: self.perf_model( + **self.nargs, + **kwargs, + **config.all_kwargs(), + ) + for config in pruned_configs + } + pruned_configs = sorted(est_timing.keys(), key=lambda x: est_timing[x])[:top_k] + return pruned_configs + + def warmup(self, *args, **kwargs): + self.nargs = dict(zip(self.arg_names, args)) + ret = [] + for config in self.prune_configs(kwargs): + ret.append(self.fn.warmup( + *args, + **kwargs, + **config.all_kwargs(), + )) + self.nargs = None + return ret + + +class Config: + """ + An object that represents a possible kernel configuration for the auto-tuner to try. + + :ivar kwargs: a dictionary of meta-parameters to pass to the kernel as keyword arguments. + :type kwargs: dict[Str, Any] + :ivar num_warps: the number of warps to use for the kernel when compiled for GPUs. For example, if + `num_warps=8`, then each kernel instance will be automatically parallelized to + cooperatively execute using `8 * 32 = 256` threads. + :type num_warps: int + :ivar num_stages: the number of stages that the compiler should use when software-pipelining loops. + Mostly useful for matrix multiplication workloads on SM80+ GPUs. + :type num_ctas: int + :ivar num_ctas: number of blocks in a block cluster. SM90+ only. + :type maxnreg: Optional[int] + :ivar maxnreg: maximum number of registers one thread can use. Corresponds + to ptx .maxnreg directive. Not supported on all platforms. + :ivar pre_hook: a function that will be called before the kernel is called. Parameters of this + function are args. + """ + + def __init__(self, kwargs, num_warps=4, num_stages=2, num_ctas=1, num_buffers_warp_spec=0, num_consumer_groups=0, + reg_dec_producer=0, reg_inc_consumer=0, maxnreg=None, pre_hook=None): + self.kwargs = kwargs + self.num_warps = num_warps + self.num_ctas = num_ctas + self.num_stages = num_stages + self.num_buffers_warp_spec = num_buffers_warp_spec + self.num_consumer_groups = num_consumer_groups + self.reg_dec_producer = reg_dec_producer + self.reg_inc_consumer = reg_inc_consumer + self.maxnreg = maxnreg + self.pre_hook = pre_hook + + def all_kwargs(self): + return { + **self.kwargs, **{ + k: v + for (k, v) in ( + ("num_warps", self.num_warps), + ("num_ctas", self.num_ctas), + ("num_stages", self.num_stages), + ("num_buffers_warp_spec", self.num_buffers_warp_spec), + ("num_consumer_groups", self.num_consumer_groups), + ("reg_dec_producer", self.reg_dec_producer), + ("reg_inc_consumer", self.reg_inc_consumer), + ("maxnreg", self.maxnreg), + ) if v is not None + } + } + + def __str__(self): + res = [] + for k, v in self.kwargs.items(): + res.append(f"{k}: {v}") + res.append(f"num_warps: {self.num_warps}") + res.append(f"num_ctas: {self.num_ctas}") + res.append(f"num_stages: {self.num_stages}") + res.append(f"num_buffers_warp_spec: {self.num_buffers_warp_spec}") + res.append(f"num_consumer_groups: {self.num_consumer_groups}") + res.append(f"reg_dec_producer: {self.reg_dec_producer}") + res.append(f"reg_inc_consumer: {self.reg_inc_consumer}") + res.append(f"maxnreg: {self.maxnreg}") + return ", ".join(res) + + +def autotune(configs, key, prune_configs_by=None, reset_to_zero=None, restore_value=None, pre_hook=None, post_hook=None, + warmup=None, rep=None, use_cuda_graph=False, do_bench=None): + """ + Decorator for auto-tuning a :code:`triton.jit`'d function. + + .. highlight:: python + .. code-block:: python + + @triton.autotune(configs=[ + triton.Config(kwargs={'BLOCK_SIZE': 128}, num_warps=4), + triton.Config(kwargs={'BLOCK_SIZE': 1024}, num_warps=8), + ], + key=['x_size'] # the two above configs will be evaluated anytime + # the value of x_size changes + ) + @triton.jit + def kernel(x_ptr, x_size, **META): + BLOCK_SIZE = META['BLOCK_SIZE'] + :note: When all the configurations are evaluated, the kernel will run multiple times. + This means that whatever value the kernel updates will be updated multiple times. + To avoid this undesired behavior, you can use the `reset_to_zero` argument, which + resets the value of the provided tensor to `zero` before running any configuration. + + If the environment variable :code:`TRITON_PRINT_AUTOTUNING` is set to + :code:`"1"`, Triton will print a message to stdout after autotuning each + kernel, including the time spent autotuning and the best configuration. + + :param configs: a list of :code:`triton.Config` objects + :type configs: list[triton.Config] + :param key: a list of argument names whose change in value will trigger the evaluation of all provided configs. + :type key: list[str] + :param prune_configs_by: a dict of functions that are used to prune configs, fields: + 'perf_model': performance model used to predicate running time with different configs, returns running time + 'top_k': number of configs to bench + 'early_config_prune'(optional): a function used to do early prune (eg, num_stages). It takes configs:List[Config] as its input, and returns pruned configs. + :param reset_to_zero: a list of argument names whose value will be reset to zero before evaluating any configs. + :type reset_to_zero: list[str] + :param restore_value: a list of argument names whose value will be restored after evaluating any configs. + :type restore_value: list[str] + :param pre_hook: a function that will be called before the kernel is called. + This overrides the default pre_hook used for 'reset_to_zero' and 'restore_value'. + 'kwargs': a dict of all arguments passed to the kernel. + 'reset_only': a boolean indicating whether the pre_hook is called to reset the values only, without a corresponding post_hook. + :type pre_hook: lambda args, reset_only + :param post_hook: a function that will be called after the kernel is called. + This overrides the default post_hook used for 'restore_value'. + 'kwargs': a dict of all arguments passed to the kernel. + 'exception': the exception raised by the kernel in case of a compilation or runtime error. + :type post_hook: lambda args, exception + :param warmup: warmup time (in ms) to pass to benchmarking (deprecated). + :type warmup: int + :param rep: repetition time (in ms) to pass to benchmarking (deprecated). + :type rep: int + :param do_bench: a benchmark function to measure the time of each run. + :type do_bench: lambda fn, quantiles + """ + + def decorator(fn): + return Autotuner(fn, fn.arg_names, configs, key, reset_to_zero, restore_value, pre_hook=pre_hook, + post_hook=post_hook, prune_configs_by=prune_configs_by, warmup=warmup, rep=rep, + use_cuda_graph=use_cuda_graph) + + return decorator + + +class Heuristics(KernelInterface): + + def __init__(self, fn, arg_names, values) -> None: + self.fn = fn + self.values = values + self.arg_names = arg_names + + def run(self, *args, **kwargs): + for v, heur in self.values.items(): + kwargs[v] = heur({**dict(zip(self.arg_names, args)), **kwargs}) + return self.fn.run(*args, **kwargs) + + +def heuristics(values): + """ + Decorator for specifying how the values of certain meta-parameters may be computed. + This is useful for cases where auto-tuning is prohibitevely expensive, or just not applicable. + + .. highlight:: python + .. code-block:: python + + @triton.heuristics(values={'BLOCK_SIZE': lambda args: 2 ** int(math.ceil(math.log2(args[1])))}) + @triton.jit + def kernel(x_ptr, x_size, **META): + BLOCK_SIZE = META['BLOCK_SIZE'] # smallest power-of-two >= x_size + :param values: a dictionary of meta-parameter names and functions that compute the value of the meta-parameter. + each such function takes a list of positional arguments as input. + :type values: dict[str, Callable[[list[Any]], Any]] + """ + + def decorator(fn): + return Heuristics(fn, fn.arg_names, values) + + return decorator diff --git a/lib/python3.12/site-packages/triton/runtime/build.py b/lib/python3.12/site-packages/triton/runtime/build.py new file mode 100644 index 0000000000000000000000000000000000000000..20da2bc25790bf07ad7b8217f5cb72285f5d6c59 --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/build.py @@ -0,0 +1,80 @@ +import contextlib +import sys +import io +import sysconfig +import os +import shutil +import subprocess +import setuptools + + +@contextlib.contextmanager +def quiet(): + old_stdout, old_stderr = sys.stdout, sys.stderr + sys.stdout, sys.stderr = io.StringIO(), io.StringIO() + try: + yield + finally: + sys.stdout, sys.stderr = old_stdout, old_stderr + + +def _build(name, src, srcdir, library_dirs, include_dirs, libraries): + suffix = sysconfig.get_config_var('EXT_SUFFIX') + so = os.path.join(srcdir, '{name}{suffix}'.format(name=name, suffix=suffix)) + # try to avoid setuptools if possible + cc = os.environ.get("CC") + if cc is None: + # TODO: support more things here. + clang = shutil.which("clang") + gcc = shutil.which("gcc") + cc = gcc if gcc is not None else clang + if cc is None: + raise RuntimeError("Failed to find C compiler. Please specify via CC environment variable.") + # This function was renamed and made public in Python 3.10 + if hasattr(sysconfig, 'get_default_scheme'): + scheme = sysconfig.get_default_scheme() + else: + scheme = sysconfig._get_default_scheme() + # 'posix_local' is a custom scheme on Debian. However, starting Python 3.10, the default install + # path changes to include 'local'. This change is required to use triton with system-wide python. + if scheme == 'posix_local': + scheme = 'posix_prefix' + py_include_dir = sysconfig.get_paths(scheme=scheme)["include"] + custom_backend_dirs = set(os.getenv(var) for var in ('TRITON_CUDACRT_PATH', 'TRITON_CUDART_PATH')) + include_dirs = include_dirs + [srcdir, py_include_dir, *custom_backend_dirs] + # for -Wno-psabi, see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111047 + cc_cmd = [cc, src, "-O3", "-shared", "-fPIC", "-Wno-psabi", "-o", so] + cc_cmd += [f'-l{lib}' for lib in libraries] + cc_cmd += [f"-L{dir}" for dir in library_dirs] + cc_cmd += [f"-I{dir}" for dir in include_dirs if dir is not None] + ret = subprocess.check_call(cc_cmd) + if ret == 0: + return so + # fallback on setuptools + extra_compile_args = [] + # extra arguments + extra_link_args = [] + # create extension module + ext = setuptools.Extension( + name=name, + language='c', + sources=[src], + include_dirs=include_dirs, + extra_compile_args=extra_compile_args + ['-O3'], + extra_link_args=extra_link_args, + library_dirs=library_dirs, + libraries=libraries, + ) + # build extension module + args = ['build_ext'] + args.append('--build-temp=' + srcdir) + args.append('--build-lib=' + srcdir) + args.append('-q') + args = dict( + name=name, + ext_modules=[ext], + script_args=args, + ) + with quiet(): + setuptools.setup(**args) + return so diff --git a/lib/python3.12/site-packages/triton/runtime/cache.py b/lib/python3.12/site-packages/triton/runtime/cache.py new file mode 100644 index 0000000000000000000000000000000000000000..82b2fea37e9bd4e18412001422b663039caf72f2 --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/cache.py @@ -0,0 +1,295 @@ +import importlib +import json +import os +import uuid +from abc import ABC, abstractmethod +from pathlib import Path +from typing import Dict, List, Optional +import base64 +import hashlib + + +def get_home_dir(): + return os.getenv("TRITON_HOME", Path.home()) + + +def default_cache_dir(): + return os.path.join(get_home_dir(), ".triton", "cache") + + +def default_override_dir(): + return os.path.join(get_home_dir(), ".triton", "override") + + +def default_dump_dir(): + return os.path.join(get_home_dir(), ".triton", "dump") + + +class CacheManager(ABC): + + def __init__(self, key): + pass + + @abstractmethod + def get_file(self, filename) -> Optional[str]: + pass + + @abstractmethod + def put(self, data, filename, binary=True) -> str: + pass + + @abstractmethod + def get_group(self, filename: str) -> Optional[Dict[str, str]]: + pass + + @abstractmethod + def put_group(self, filename: str, group: Dict[str, str]): + pass + + +class FileCacheManager(CacheManager): + + def __init__(self, key, override=False, dump=False): + self.key = key + self.lock_path = None + if dump: + self.cache_dir = os.getenv("TRITON_DUMP_DIR", "").strip() or default_dump_dir() + self.cache_dir = os.path.join(self.cache_dir, self.key) + self.lock_path = os.path.join(self.cache_dir, "lock") + os.makedirs(self.cache_dir, exist_ok=True) + elif override: + self.cache_dir = os.getenv("TRITON_OVERRIDE_DIR", "").strip() or default_override_dir() + self.cache_dir = os.path.join(self.cache_dir, self.key) + else: + # create cache directory if it doesn't exist + self.cache_dir = os.getenv("TRITON_CACHE_DIR", "").strip() or default_cache_dir() + if self.cache_dir: + self.cache_dir = os.path.join(self.cache_dir, self.key) + self.lock_path = os.path.join(self.cache_dir, "lock") + os.makedirs(self.cache_dir, exist_ok=True) + else: + raise RuntimeError("Could not create or locate cache dir") + + def _make_path(self, filename) -> str: + return os.path.join(self.cache_dir, filename) + + def has_file(self, filename) -> bool: + if not self.cache_dir: + raise RuntimeError("Could not create or locate cache dir") + return os.path.exists(self._make_path(filename)) + + def get_file(self, filename) -> Optional[str]: + if self.has_file(filename): + return self._make_path(filename) + else: + return None + + def get_group(self, filename: str) -> Optional[Dict[str, str]]: + grp_filename = f"__grp__{filename}" + if not self.has_file(grp_filename): + return None + grp_filepath = self._make_path(grp_filename) + with open(grp_filepath) as f: + grp_data = json.load(f) + child_paths = grp_data.get("child_paths", None) + # Invalid group data. + if child_paths is None: + return None + result = {} + for c, p in child_paths.items(): + if os.path.exists(p): + result[c] = p + return result + + # Note a group of pushed files as being part of a group + def put_group(self, filename: str, group: Dict[str, str]) -> str: + if not self.cache_dir: + raise RuntimeError("Could not create or locate cache dir") + grp_contents = json.dumps({"child_paths": group}) + grp_filename = f"__grp__{filename}" + return self.put(grp_contents, grp_filename, binary=False) + + def put(self, data, filename, binary=True) -> str: + if not self.cache_dir: + raise RuntimeError("Could not create or locate cache dir") + binary = isinstance(data, bytes) + if not binary: + data = str(data) + assert self.lock_path is not None + filepath = self._make_path(filename) + # Random ID to avoid any collisions + rnd_id = str(uuid.uuid4()) + # we use the PID in case a bunch of these around so we can see what PID made it + pid = os.getpid() + # use temp dir to be robust against program interruptions + temp_dir = os.path.join(self.cache_dir, f"tmp.pid_{pid}_{rnd_id}") + os.makedirs(temp_dir, exist_ok=True) + temp_path = os.path.join(temp_dir, filename) + + mode = "wb" if binary else "w" + with open(temp_path, mode) as f: + f.write(data) + # Replace is guaranteed to be atomic on POSIX systems if it succeeds + # so filepath cannot see a partial write + os.replace(temp_path, filepath) + os.removedirs(temp_dir) + return filepath + + +class RemoteCacheBackend: + """ + A backend implementation for accessing a remote/distributed cache. + """ + + def __init__(self, key: str): + pass + + @abstractmethod + def get(self, filenames: List[str]) -> Dict[str, bytes]: + pass + + @abstractmethod + def put(self, filename: str, data: bytes): + pass + + +class RedisRemoteCacheBackend(RemoteCacheBackend): + + def __init__(self, key): + import redis + self._key = key + self._key_fmt = os.environ.get("TRITON_REDIS_KEY_FORMAT", "triton:{key}:{filename}") + self._redis = redis.Redis( + host=os.environ.get("TRITON_REDIS_HOST", "localhost"), + port=int(os.environ.get("TRITON_REDIS_PORT", 6379)), + ) + + def _get_key(self, filename: str) -> str: + return self._key_fmt.format(key=self._key, filename=filename) + + def get(self, filenames: List[str]) -> Dict[str, str]: + results = self._redis.mget([self._get_key(f) for f in filenames]) + return {filename: result for filename, result in zip(filenames, results) if result is not None} + + def put(self, filename: str, data: bytes) -> Dict[str, bytes]: + self._redis.set(self._get_key(filename), data) + + +class RemoteCacheManager(CacheManager): + + def __init__(self, key, override=False, dump=False): + # Setup backend pointed too by `TRITON_REMOTE_CACHE_BACKEND`. + remote_cache_manager = os.environ["TRITON_REMOTE_CACHE_BACKEND"] + module_path, clz_nme = remote_cache_manager.split(":") + module = importlib.import_module(module_path) + remote_cache_cls = getattr(module, clz_nme) + self._backend = remote_cache_cls(key) + + self._override = override + self._dump = dump + + # Use a `FileCacheManager` to materialize remote cache paths locally. + self._file_cache_manager = FileCacheManager(key, override=override, dump=dump) + + def _materialize(self, filename: str, data: bytes): + # We use a backing `FileCacheManager` to provide the materialized data. + return self._file_cache_manager.put(data, filename, binary=True) + + def get_file(self, filename: str) -> Optional[str]: + # We don't handle the dump/override cases. + if self._dump or self._override: + return self._file_cache_manager.get_file(filename) + + # We always check the remote cache backend -- even if our internal file- + # based cache has the item -- to make sure LRU accounting works as + # expected. + results = self._backend.get([filename]) + if len(results) == 0: + return None + (_, data), = results.items() + return self._materialize(filename, data) + + def put(self, data, filename: str, binary=True) -> str: + # We don't handle the dump/override cases. + if self._dump or self._override: + return self._file_cache_manager.put(data, filename, binary=binary) + + if not isinstance(data, bytes): + data = str(data).encode("utf-8") + self._backend.put(filename, data) + return self._materialize(filename, data) + + def get_group(self, filename: str) -> Optional[Dict[str, str]]: + # We don't handle the dump/override cases. + if self._dump or self._override: + return self._file_cache_manager.get_group(filename) + + grp_filename = f"__grp__{filename}" + grp_filepath = self.get_file(grp_filename) + if grp_filepath is None: + return None + with open(grp_filepath) as f: + grp_data = json.load(f) + child_paths = grp_data.get("child_paths", None) + + result = None + + # Found group data. + if child_paths is not None: + result = {} + for child_path, data in self._backend.get(child_paths).items(): + result[child_path] = self._materialize(child_path, data) + + return result + + def put_group(self, filename: str, group: Dict[str, str]): + # We don't handle the dump/override cases. + if self._dump or self._override: + return self._file_cache_manager.put_group(filename, group) + + grp_contents = json.dumps({"child_paths": sorted(list(group.keys()))}) + grp_filename = f"__grp__{filename}" + return self.put(grp_contents, grp_filename) + + +__cache_cls = FileCacheManager +__cache_cls_nme = "DEFAULT" + + +def _base64(key): + # Assume key is a hex string. + return base64.urlsafe_b64encode(bytes.fromhex(key)).decode("utf-8").rstrip("=") + + +def get_cache_manager(key) -> CacheManager: + import os + + user_cache_manager = os.environ.get("TRITON_CACHE_MANAGER", None) + global __cache_cls + global __cache_cls_nme + + if user_cache_manager is not None and user_cache_manager != __cache_cls_nme: + module_path, clz_nme = user_cache_manager.split(":") + module = importlib.import_module(module_path) + __cache_cls = getattr(module, clz_nme) + __cache_cls_nme = user_cache_manager + + return __cache_cls(_base64(key)) + + +def get_override_manager(key) -> CacheManager: + return __cache_cls(_base64(key), override=True) + + +def get_dump_manager(key) -> CacheManager: + return __cache_cls(_base64(key), dump=True) + + +def make_so_cache_key(version_hash, signature, constants, ids, **kwargs): + # Get unique key for the compiled code + signature = {k: 'ptr' if v[0] == '*' else v for k, v in signature.items()} + key = f"{version_hash}-{''.join(signature.values())}-{constants}-{ids}" + for kw in kwargs: + key = f"{key}-{kwargs.get(kw)}" + key = hashlib.sha256(key.encode("utf-8")).hexdigest() + return _base64(key) diff --git a/lib/python3.12/site-packages/triton/runtime/driver.py b/lib/python3.12/site-packages/triton/runtime/driver.py new file mode 100644 index 0000000000000000000000000000000000000000..c3b97a764145490b8c955cc37fe366bb755176a4 --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/driver.py @@ -0,0 +1,60 @@ +from ..backends import backends +from ..backends import DriverBase + + +def _create_driver(): + actives = [x.driver for x in backends.values() if x.driver.is_active()] + if len(actives) != 1: + raise RuntimeError(f"{len(actives)} active drivers ({actives}). There should only be one.") + return actives[0]() + + +class LazyProxy: + + def __init__(self, init_fn): + self._init_fn = init_fn + self._obj = None + + def _initialize_obj(self): + if self._obj is None: + self._obj = self._init_fn() + + def __getattr__(self, name): + self._initialize_obj() + return getattr(self._obj, name) + + def __setattr__(self, name, value): + if name in ["_init_fn", "_obj"]: + super().__setattr__(name, value) + else: + self._initialize_obj() + setattr(self._obj, name, value) + + def __delattr__(self, name): + self._initialize_obj() + delattr(self._obj, name) + + def __repr__(self): + if self._obj is None: + return f"<{self.__class__.__name__} for {self._init_fn} not yet initialized>" + return repr(self._obj) + + def __str__(self): + self._initialize_obj() + return str(self._obj) + + +class DriverConfig: + + def __init__(self): + self.default = LazyProxy(_create_driver) + self.active = self.default + + def set_active(self, driver: DriverBase): + self.active = driver + + def reset_active(self): + self.active = self.default + + +driver = DriverConfig() diff --git a/lib/python3.12/site-packages/triton/runtime/errors.py b/lib/python3.12/site-packages/triton/runtime/errors.py new file mode 100644 index 0000000000000000000000000000000000000000..4dce9176709a91675921af217b55f530ff57ddc9 --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/errors.py @@ -0,0 +1,26 @@ +from ..errors import TritonError +from typing import Optional + + +class InterpreterError(TritonError): + + def __init__(self, error_message: Optional[str] = None): + self.error_message = error_message + + def __str__(self) -> str: + return self.error_message or "" + + +class OutOfResources(TritonError): + + def __init__(self, required, limit, name): + self.required = required + self.limit = limit + self.name = name + + def __str__(self) -> str: + return f"out of resource: {self.name}, Required: {self.required}, Hardware limit: {self.limit}. Reducing block sizes or `num_stages` may help." + + def __reduce__(self): + # this is necessary to make CompilationError picklable + return (type(self), (self.required, self.limit, self.name)) diff --git a/lib/python3.12/site-packages/triton/runtime/interpreter.py b/lib/python3.12/site-packages/triton/runtime/interpreter.py new file mode 100644 index 0000000000000000000000000000000000000000..0aeaff73a4ea5eaf805771a02b43d1df52964747 --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/interpreter.py @@ -0,0 +1,1235 @@ +import ast +import textwrap +import inspect +from typing import Tuple + +import math +import numpy as np + +import triton +import triton.language as tl +from dataclasses import dataclass +from .errors import InterpreterError +from functools import partial +from .._C.libtriton import interpreter as _interpreter +from .._C.libtriton import ir as _ir + + +class TensorHandle: + + def __init__(self, data, dtype): + ''' + data: numpy array + dtype: triton type, either pointer_type or scalar_type. + we don't store block_type here because the shape information is already availale in the data field + attr: a dictionary of attributes + ''' + self.data = data + self.dtype = dtype + self.attr = {} + + def __bool__(self): + return bool(self.data.all()) + + def get_element_ty(self): + dtype = self.dtype + while hasattr(dtype, "element_ty"): + dtype = dtype.element_ty + return dtype + + def clone(self): + return TensorHandle(self.data.copy(), self.dtype) + + def set_attr(self, key, value): + self.attr[key] = value + + +class BlockPointerHandle: + + def __init__(self, base, shape, strides, offsets, tensor_shape, order): + self.base = base + self.shape = shape + self.strides = strides + self.offsets = offsets + self.tensor_shape = tensor_shape + self.order = order + + def materialize_pointers(self, boundary_check): + dtype_tt = self.base.get_element_ty() + n_bytes = dtype_tt.primitive_bitwidth // 8 + tensor_shape = self.tensor_shape + ptrs = np.broadcast_to(self.base.data, self.tensor_shape) + masks = np.ones(self.tensor_shape, dtype=bool) + for dim in range(len(tensor_shape)): + bcast_dims = [1] * len(tensor_shape) + bcast_dims[dim] = tensor_shape[dim] + off = (self.offsets[dim].data + np.arange(tensor_shape[dim])).reshape(bcast_dims) + ptrs = ptrs + (n_bytes * off * self.strides[dim].data).astype(np.uint64) + if dim in boundary_check: + masks = np.logical_and(masks, off < self.shape[dim].data) + ptrs = TensorHandle(ptrs, self.base.dtype.scalar) + return ptrs, masks + + +@dataclass(frozen=True) +class InterpreterOptions: + extern_libs: dict = None + debug: bool = False + sanitize_overflow: bool = True + arch: str = None + supported_fp8_dtypes: Tuple[str] = ("fp8e5", "fp8e5b16", "fp8e4nv", "fp8e4b8", "fp8e4b15") + deprecated_fp8_dtypes: Tuple[str] = () + default_dot_input_precision: str = "tf32" + allowed_dot_input_precisions: Tuple[str] = ("tf32", "tf32x3", "ieee") + max_num_imprecise_acc_default: int = 0 + backend_name: str = "interpreter" + + +def _get_signed_np_dtype(dtype): + if dtype == np.uint8: + return np.int8 + if dtype == np.uint16: + return np.int16 + if dtype == np.uint32: + return np.int32 + if dtype == np.uint64: + return np.int64 + return dtype + + +def _get_np_dtype(tt_dtype): + if isinstance(tt_dtype, tl.pointer_type): + return np.dtype(np.uint64) + np_types = { + tl.int1: np.dtype(bool), + tl.float16: np.dtype(np.float16), + tl.float32: np.dtype(np.float32), + tl.float64: np.dtype(np.float64), + tl.int8: np.dtype(np.int8), + tl.uint8: np.dtype(np.uint8), + tl.int16: np.dtype(np.int16), + tl.uint16: np.dtype(np.uint16), + tl.int32: np.dtype(np.int32), + tl.uint32: np.dtype(np.uint32), + tl.int64: np.dtype(np.int64), + tl.uint64: np.dtype(np.uint64), + # bfloat16 types are stored as uint16 + tl.bfloat16: np.dtype(np.uint16), + # float8 types are stored as uint8 + tl.float8e5: np.dtype(np.uint8), + tl.float8e5b16: np.dtype(np.uint8), + tl.float8e4nv: np.dtype(np.uint8), + tl.float8e4b8: np.dtype(np.uint8), + tl.float8e4b15: np.dtype(np.uint8), + } + if isinstance(tt_dtype, tl.block_type): + if isinstance(tt_dtype.element_ty, tl.pointer_type): + return np.dtype(np.uint64) + return np_types[tt_dtype.element_ty] + return np_types[tt_dtype] + + +def _convert_float(input, input_dtype, output_dtype, rounding_mode): + input_uint_dtype = getattr(np, f"uint{input_dtype.primitive_bitwidth}") + output_unint_dtype = getattr(np, f"uint{output_dtype.primitive_bitwidth}") + input_bin = np.frombuffer(input.tobytes(), dtype=input_uint_dtype) + sign = (input_bin >> (input_dtype.primitive_bitwidth - 1)) & 0x01 + input_exponent_width = input_dtype.primitive_bitwidth - input_dtype.fp_mantissa_width - 1 + output_exponent_width = output_dtype.primitive_bitwidth - output_dtype.fp_mantissa_width - 1 + significand = input_bin & ((1 << input_dtype.fp_mantissa_width) - 1) + bias_input = input_dtype.exponent_bias + bias_output = output_dtype.exponent_bias + exponent = ((input_bin >> input_dtype.fp_mantissa_width) & ((1 << input_exponent_width) - 1)).astype(np.int32) + subnormal_index = exponent == 0 + if np.any(subnormal_index): + # Credit to Phil: phil@openai.com + # subnormal repr: ((-1.0)**sign) * (2.0**(1 - exp_bias)) * (2^(m0) + 2^(m1) + ... + 2^(mn)) + # where m0, m1, ..., mn are the 1-bit of the mantissa + # convert it to normal repr: ((-1.0)**sign) * (2.0**(1 + m0 - exp_bias)) * (1 + 2^(m1 - m0) + ... + 2^(mn - m0)) + bit_pos = np.zeros_like(input_bin, dtype=np.int32) + # Find the most significant bit of the mantissa in the significand + for i in range(input_dtype.fp_mantissa_width): + bit_index = ((significand >> i) & 0x01) + # pos should be >= 1 + bit_pos[bit_index == 1] = input_dtype.fp_mantissa_width - i + zero_significand_index = significand == 0 + exponent[subnormal_index] = 1 - bit_pos[subnormal_index] + # 0 significand and subnormal should be treated as 0 + exponent[zero_significand_index & subnormal_index] = bias_input - bias_output + significand[subnormal_index] = (significand[subnormal_index] << bit_pos[subnormal_index]) & ( + (1 << input_dtype.fp_mantissa_width) - 1) + # Prevent overflow and underflow + exponent_output = np.maximum(0, np.minimum((exponent - bias_input + bias_output), (1 << output_exponent_width) - 1)) + exponent_output = exponent_output.astype(output_unint_dtype) + sign_output = sign.astype(output_unint_dtype) + if input_dtype.primitive_bitwidth > output_dtype.primitive_bitwidth: # Downcast + significand_output = (significand >> (input_dtype.fp_mantissa_width - output_dtype.fp_mantissa_width)) & ( + (1 << output_dtype.fp_mantissa_width) - 1) + if rounding_mode == _ir.ROUNDING_MODE.RTNE: # Round to nearst even + # find the cut-off bit + cut_off = significand & (1 << (input_dtype.fp_mantissa_width - output_dtype.fp_mantissa_width - 1)) + significand_output = significand_output + (cut_off > 0) + significand_output = significand_output.astype(output_unint_dtype) + else: # Upcast + significand_output = (significand.astype(output_unint_dtype) << + (output_dtype.fp_mantissa_width - input_dtype.fp_mantissa_width)) & ( + (1 << output_dtype.fp_mantissa_width) - 1) + subnormal_index = exponent_output == 0 + if np.any(subnormal_index): # underflow + # normal repr: ((-1.0)**sign) * (2.0**(exp - exp_bias_input)) * (1 + 2^(m0) + 2^(m1) + ... + 2^(mn)) + # where m0, m1, ..., mn are the 1-bit of the mantissa + # shift = (1 - exp_bias_output) - (exp - exp_bias_input) + # convert it to subnormal repr: ((-1.0)**sign) * (2.0**(1 - exp_bias_output)) * (2^(-shift) + 2^(m0 - shift) + 2^(m1 - shift) + ... + 2^(mn - shift)) + exponent = ((input_bin >> input_dtype.fp_mantissa_width) & ((1 << input_exponent_width) - 1)).astype(np.int32) + non_zero_exponent_index = exponent != 0 + # If the original exponent is not zero, we still need to shift the significand and consider the 1.0 part in mantissa + subnormal_index = subnormal_index & non_zero_exponent_index + shift = np.zeros_like(input_bin, dtype=np.int32) + shift[subnormal_index] = (1 - bias_output) - (exponent[subnormal_index] - bias_input) + significand_output[subnormal_index] = (significand_output[subnormal_index] >> shift[subnormal_index]) | ( + 1 << (output_dtype.fp_mantissa_width - shift[subnormal_index])) + output = (sign_output << (output_dtype.primitive_bitwidth - 1)) | ( + exponent_output << output_dtype.fp_mantissa_width) | significand_output + return output.reshape(input.shape) + + +def _erf(x): + # Numpy does not support erf + return math.erf(x) + + +def _umulhi_64(a, b): + # Numpy does not support 128-bit multiplication + # So we have to implement it manually + return (int(a) * int(b)) >> 64 + + +np_erf_fp32 = np.vectorize(_erf, otypes=[np.float32]) +np_erf_fp64 = np.vectorize(_erf, otypes=[np.float64]) +np_umulhi_u64 = np.vectorize(_umulhi_64, otypes=[np.uint64]) + + +class ExtraFunctions: + + @staticmethod + def _convert_custom_types(input, dst_ty, fp_downcast_rounding, _builder): + return tl.tensor(_builder.create_fp_to_fp(input.handle, dst_ty, fp_downcast_rounding), dst_ty) + + +class InterpreterBuilder: + ir_sem_to_interpreter_sem = { + _ir.MEM_SEMANTIC.ACQUIRE: _interpreter.MEM_SEMANTIC.ACQUIRE, + _ir.MEM_SEMANTIC.RELEASE: _interpreter.MEM_SEMANTIC.RELEASE, + _ir.MEM_SEMANTIC.RELAXED: _interpreter.MEM_SEMANTIC.RELAXED, + _ir.MEM_SEMANTIC.ACQUIRE_RELEASE: _interpreter.MEM_SEMANTIC.ACQUIRE_RELEASE, + } + + ir_rmw_op_to_interpreter_rmw_op = { + _ir.ATOMIC_OP.ADD: _interpreter.RMW_OP.ADD, + _ir.ATOMIC_OP.FADD: _interpreter.RMW_OP.FADD, + _ir.ATOMIC_OP.MIN: _interpreter.RMW_OP.MIN, + _ir.ATOMIC_OP.UMIN: _interpreter.RMW_OP.UMIN, + _ir.ATOMIC_OP.MAX: _interpreter.RMW_OP.MAX, + _ir.ATOMIC_OP.UMAX: _interpreter.RMW_OP.UMAX, + _ir.ATOMIC_OP.AND: _interpreter.RMW_OP.AND, + _ir.ATOMIC_OP.OR: _interpreter.RMW_OP.OR, + _ir.ATOMIC_OP.XOR: _interpreter.RMW_OP.XOR, + _ir.ATOMIC_OP.XCHG: _interpreter.RMW_OP.XCHG, + } + + def __init__(self) -> None: + self.arch = None + self.options = InterpreterOptions() + self.codegen_fns = {} + self.codegen_fns["convert_custom_types"] = ExtraFunctions._convert_custom_types + self.codegen_fns["min_dot_size"] = lambda lhsType, rhsType: (16, 16, 16) + + def set_grid_idx(self, x, y, z): + if not x < self.grid_dim[0]: + raise ValueError("x >= grid_dim[0]") + if not y < self.grid_dim[1]: + raise ValueError("y >= grid_dim[1]") + if not z < self.grid_dim[2]: + raise ValueError("z >= grid_dim[2]") + self.grid_idx = (x, y, z) + + def set_grid_dim(self, nx, ny, nz): + self.grid_dim = (nx, ny, nz) + + # constants + + def get_half_ty(self): + return tl.float16 + + def get_bf16_ty(self): + return tl.bfloat16 + + def get_float_ty(self): + return tl.float32 + + def get_double_ty(self): + return tl.float64 + + def get_int8_ty(self): + return tl.int8 + + def get_uint8_ty(self): + return tl.uint8 + + def get_int16_ty(self): + return tl.int16 + + def get_uint16_ty(self): + return tl.uint16 + + def get_int32_ty(self): + return tl.int32 + + def get_uint32_ty(self): + return tl.uint32 + + def get_int64_ty(self): + return tl.int64 + + def get_uint64_ty(self): + return tl.uint64 + + def get_fp8e4nv_ty(self): + return tl.float8e4nv + + def get_fp8e4b15_ty(self): + return tl.float8e4b15 + + def get_fp8e4b8_ty(self): + return tl.float8e4b8 + + def get_fp8e5_ty(self): + return tl.float8e5 + + def get_fp8e5b16_ty(self): + return tl.float8e5b16 + + def get_ptr_ty(self, elt_ty, addr_space): + return tl.pointer_type(elt_ty, addr_space) + + def get_block_ty(self, dtype, shape): + return tl.block_type(dtype, shape) + + def get_int1(self, value): + return TensorHandle(np.array([value], dtype=np.bool_), tl.int1) + + def get_uint8(self, value): + return TensorHandle(np.array([value], dtype=np.uint8), tl.uint8) + + def get_int8(self, value): + return TensorHandle(np.array([value], dtype=np.int8), tl.int8) + + def get_uint16(self, value): + return TensorHandle(np.array([value], dtype=np.uint16), tl.uint16) + + def get_int16(self, value): + return TensorHandle(np.array([value], dtype=np.int16), tl.int16) + + def get_uint32(self, value): + return TensorHandle(np.array([value], dtype=np.uint32), tl.uint32) + + def get_int32(self, value): + return TensorHandle(np.array([value], dtype=np.int32), tl.int32) + + def get_uint64(self, value): + return TensorHandle(np.array([value], dtype=np.uint64), tl.uint64) + + def get_int64(self, value): + return TensorHandle(np.array([value], dtype=np.int64), tl.int64) + + def get_fp16(self, value): + return TensorHandle(np.array([value], dtype=np.float16), tl.float16) + + def get_fp32(self, value): + return TensorHandle(np.array([value], dtype=np.float32), tl.float32) + + def get_fp64(self, value): + return TensorHandle(np.array([value], dtype=np.float64), tl.float64) + + def get_null_value(self, type): + return TensorHandle(np.array([0], dtype=_get_np_dtype(type)), type) + + # programming model + def create_get_program_id(self, axis): + if self.grid_idx is None: + raise ValueError("grid_idx is None") + return TensorHandle(np.array([self.grid_idx[axis]], dtype=np.int32), tl.int32) + + def create_get_num_programs(self, axis): + return TensorHandle(np.array([self.grid_dim[axis]], dtype=np.int32), tl.int32) + + # memory ops + def create_load(self, ptr, _0, _1, is_volatile): + mask = TensorHandle(np.ones_like(ptr.data, dtype=bool), tl.int1) + other = None + return self.create_masked_load(ptr, mask, other, _0, _1, is_volatile) + + def create_store(self, ptr, val, _0, _1): + mask = TensorHandle(np.ones_like(ptr.data, dtype=bool), tl.int1) + return self.create_masked_store(ptr, val, mask, None, None) + + def create_masked_load(self, ptrs, mask, other, cache_modifier, eviction_policy, is_volatile): + dtype_tt = ptrs.get_element_ty() + dtype_np = _get_np_dtype(dtype_tt) + if other is None: + other = TensorHandle(np.zeros_like(ptrs.data, dtype=dtype_np), dtype_tt) + ret = _interpreter.load(ptrs.data, mask.data, other.data, dtype_np) + return TensorHandle(ret, dtype_tt) + + def create_masked_store(self, ptrs, value, mask, cache_modifier, eviction_policy): + return _interpreter.store(ptrs.data, value.data, mask.data) + + # casting ops + def cast_impl(self, src, dst_type): + src_element_type = src.dtype.scalar + dst_element_type = dst_type.scalar + if (src_element_type == tl.bfloat16 and dst_element_type == tl.float32) or \ + (src_element_type == tl.float32 and dst_element_type == tl.bfloat16): + data = _convert_float(src.data, src_element_type, dst_element_type, None).view(_get_np_dtype(dst_type)) + return TensorHandle(data, dst_type.scalar) + else: + return TensorHandle(src.data.astype(_get_np_dtype(dst_type)), dst_type.scalar) + + create_si_to_fp = lambda self, src, dst_type: self.cast_impl(src, dst_type) + create_ui_to_fp = lambda self, src, dst_type: self.cast_impl(src, dst_type) + create_fp_to_si = lambda self, src, dst_type: self.cast_impl(src, dst_type) + create_fp_to_ui = lambda self, src, dst_type: self.cast_impl(src, dst_type) + create_fp_ext = lambda self, src, dst_type: self.cast_impl(src, dst_type) + create_fp_trunc = lambda self, src, dst_type: self.cast_impl(src, dst_type) + create_int_cast = lambda self, src, dst_type, is_signed: self.cast_impl(src, dst_type) + + def create_fp_to_fp(self, src, dst_type, rounding_mode): + src_element_type = src.dtype.scalar + dst_element_type = dst_type.scalar + data = _convert_float(src.data, src_element_type, dst_element_type, rounding_mode).view(_get_np_dtype(dst_type)) + return TensorHandle(data, dst_type.scalar) + + def create_bitcast(self, src, dst_type): + return TensorHandle(src.data.view(_get_np_dtype(dst_type)), dst_type.scalar) + + # binary operators + def binary_op(self, lhs, rhs, op): + return TensorHandle(op(lhs.data, rhs.data), lhs.dtype.scalar) + + create_fadd = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.add) + create_fmul = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.multiply) + create_fdiv = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.divide) + create_frem = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.remainder) + create_fsub = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.subtract) + create_mul = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.multiply) + create_precise_divf = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.divide) + create_sdiv = lambda self, lhs, rhs: self.create_idiv(lhs, rhs) + create_udiv = lambda self, lhs, rhs: self.create_idiv(lhs, rhs) + # LLVM has 'numpy.fmod', not 'numpy.remainder', semantics on integer remainders. + create_srem = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.fmod) + create_urem = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.fmod) + create_add = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.add) + create_sub = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.subtract) + create_shl = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.left_shift) + create_lshr = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.right_shift) + create_minsi = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.minimum) + create_minui = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.minimum) + create_minimumf = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.minimum) + create_minnumf = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.minimum) + create_maxsi = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.maximum) + create_maxui = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.maximum) + create_maximumf = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.maximum) + create_maxnumf = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.maximum) + create_icmpSLE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less_equal) + create_icmpSLT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less) + create_icmpSGE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater_equal) + create_icmpSGT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater) + create_icmpULE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less_equal) + create_icmpULT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less) + create_icmpUGE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater_equal) + create_icmpUGT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater) + create_icmpEQ = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.equal) + create_icmpNE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.not_equal) + create_fcmpOLT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less) + create_fcmpOGT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater) + create_fcmpOLE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less_equal) + create_fcmpOGE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater_equal) + create_fcmpOEQ = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.equal) + create_fcmpONE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.not_equal) + create_fcmpULT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less) + create_fcmpUGT = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater) + create_fcmpULE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.less_equal) + create_fcmpUGE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.greater_equal) + create_fcmpUEQ = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.equal) + create_fcmpUNE = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.not_equal) + create_and = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.bitwise_and) + create_xor = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.bitwise_xor) + create_or = lambda self, lhs, rhs: self.binary_op(lhs, rhs, np.bitwise_or) + create_int_to_ptr = create_bitcast + create_ptr_to_int = create_bitcast + + def create_idiv(self, lhs, rhs): + # Triton has IEEE, not numpy/torch, semantics for %, and those carry + # through to //, so we have to use a nonstandard expression to get a + # reference result for //. + return TensorHandle((lhs.data - np.fmod(lhs.data, rhs.data)) // rhs.data, lhs.dtype.scalar) + + def create_ashr(self, lhs, rhs): + # Triton's rshift operator depends on the signedness of the left operand + lhs_dtype = _get_signed_np_dtype(lhs.data.dtype) + rhs_dtype = _get_signed_np_dtype(rhs.data.dtype) + lhs.data = lhs.data.astype(lhs_dtype) + rhs.data = rhs.data.astype(rhs_dtype) + return self.binary_op(lhs, rhs, np.right_shift) + + def create_umulhi(self, lhs, rhs): + dtype = lhs.data.dtype + if dtype == np.int64 or dtype == np.uint64: + return TensorHandle(np_umulhi_u64(lhs.data, rhs.data), lhs.dtype.scalar) + else: + compute_dtype = getattr(np, f"uint{dtype.itemsize * 8 * 2}") + lhs_data = lhs.data.astype(compute_dtype) + rhs_data = rhs.data.astype(compute_dtype) + ret_data = np.multiply(lhs_data, rhs_data) >> (dtype.itemsize * 8) + return TensorHandle(ret_data.astype(dtype), lhs.dtype.scalar) + + # ternary functions + def ternary_op(self, lhs, rhs, other, op): + return TensorHandle(op(lhs.data, rhs.data, other.data), other.dtype.scalar) + + create_clampf = lambda self, arg, lo, hi, propagate_nans: self.ternary_op(arg, lo, hi, np.clip) + create_select = lambda self, cond, lhs, rhs: self.ternary_op(cond, lhs, rhs, np.where) + + def create_fma(self, x, y, z): + return TensorHandle(x.data * y.data + z.data, z.dtype.scalar) + + # unary functions + def unary_op(self, arg, op): + return TensorHandle(op(arg.data), arg.dtype.scalar) + + def create_fabs(self, arg): + # Mask out the sign bit based on the primitive length + dtype_tt = arg.dtype + mask_bitwidth = dtype_tt.primitive_bitwidth - 1 + np_uint_dtype = getattr(np, f"uint{dtype_tt.primitive_bitwidth}") + data = arg.data.view(np_uint_dtype) + mask = (1 << mask_bitwidth) - 1 + ret = (data & mask).view(_get_np_dtype(dtype_tt)) + return TensorHandle(ret, arg.dtype.scalar) + + create_cos = lambda self, arg: self.unary_op(arg, np.cos) + create_exp = lambda self, arg: self.unary_op(arg, np.exp) + create_exp2 = lambda self, arg: self.unary_op(arg, np.exp2) + create_iabs = lambda self, arg: self.unary_op(arg, np.abs) + create_floor = lambda self, arg: self.unary_op(arg, np.floor) + create_ceil = lambda self, arg: self.unary_op(arg, np.ceil) + create_log = lambda self, arg: self.unary_op(arg, np.log) + create_log2 = lambda self, arg: self.unary_op(arg, np.log2) + create_precise_sqrt = lambda self, arg: self.unary_op(arg, np.sqrt) + create_sqrt = lambda self, arg: self.unary_op(arg, np.sqrt) + create_sin = lambda self, arg: self.unary_op(arg, np.sin) + + def create_erf(self, arg): + ret = np_erf_fp32(arg.data) if arg.data.dtype == np.float32 else np_erf_fp64(arg.data) + return TensorHandle(ret, arg.dtype.scalar) + + def create_rsqrt(self, arg): + return TensorHandle(1 / np.sqrt(arg.data), arg.dtype.scalar) + + # tensor operators + create_reshape = lambda self, arg, shape, allow_reorder: TensorHandle(arg.data.reshape(shape), arg.dtype.scalar) + + def create_trans(self, arg, perm): + return TensorHandle(np.transpose(arg.data, perm), arg.dtype.scalar) + + def create_dot(self, a, b, d, input_precision, max_num_imprecise_acc): + a_data = a.data + b_data = b.data + if (a.dtype.primitive_bitwidth == 8 and a.dtype.is_floating()) or \ + (b.dtype.primitive_bitwidth == 8 and b.dtype.is_floating()): + a_data = _convert_float(a_data, a.dtype, tl.float16, None).view(np.float16) + b_data = _convert_float(b_data, b.dtype, tl.float16, None).view(np.float16) + return TensorHandle(np.matmul(a_data, b_data, dtype=d.data.dtype) + d.data, d.dtype.scalar) + + def create_make_range(self, start, stop): + return TensorHandle(np.arange(start, stop, dtype=np.int32), tl.int32) + + def create_histogram(self, data, bins): + return TensorHandle(np.histogram(data.data, bins=bins, range=(0, bins))[0], tl.int32) + + # pointer arithmetic + + def create_addptr(self, ptr, offset): + dtype_tt = ptr.get_element_ty() + element_bitwidth = dtype_tt.primitive_bitwidth + # int1's bitwidth is 1, but we need to use 8 for pointer arithmetic + element_bytewidth = max(1, element_bitwidth // 8) + return TensorHandle(ptr.data + element_bytewidth * offset.data.astype(np.uint64), ptr.dtype) + + def create_tensor_pointer_load(self, ptr, boundary_check, padding_option, cache_modifier, eviction_policy, + is_volatile): + ptrs, masks = ptr.materialize_pointers(boundary_check) + dtype_tt = ptrs.get_element_ty() + dtype_np = _get_np_dtype(dtype_tt) + if padding_option is None: + other = None + elif padding_option == _ir.PADDING_OPTION.PAD_ZERO: + other = TensorHandle(np.zeros_like(ptrs.data, dtype=dtype_np), dtype_tt) + elif padding_option == _ir.PADDING_OPTION.PAD_NAN: + other = TensorHandle(np.full_like(ptrs.data, float('nan'), dtype=dtype_np), dtype_tt) + else: + raise ValueError(f"unsupported padding option {padding_option}") + return self.create_masked_load(ptrs, masks, other, cache_modifier, eviction_policy, is_volatile) + + def create_tensor_pointer_store(self, ptr, value, boundary_check, cache_modifier, eviction_policy): + ptrs, masks = ptr.materialize_pointers(boundary_check) + return self.create_masked_store(ptrs, value, masks, cache_modifier, eviction_policy) + + def create_expand_dims(self, arg, axis): + return TensorHandle(np.expand_dims(arg.data, axis), arg.dtype.scalar) + + def create_broadcast(self, arg, shape): + return TensorHandle(np.broadcast_to(arg.data, shape), arg.dtype.scalar) + + def create_cat(self, lhs, rhs): + return TensorHandle(np.concatenate([lhs.data, rhs.data]), lhs.dtype.scalar) + + def create_join(self, lhs, rhs): + # Triton only supports joining two original tensors into a new one along the last axis + return TensorHandle(np.stack([lhs.data, rhs.data], axis=-1), lhs.dtype.scalar) + + def create_split(self, val): + # Triton only supports splitting the original tensor into two along the last axis + return (TensorHandle(val.data[..., 0], val.dtype.scalar), TensorHandle(val.data[..., 1], val.dtype.scalar)) + + def create_splat(self, arg, shape): + if isinstance(arg.dtype, tl.block_type): + return TensorHandle(np.full(shape, arg.data[0], dtype=_get_np_dtype(arg.dtype)), arg.dtype.scalar) + else: # scalar + return TensorHandle(np.full(shape, arg.data, dtype=_get_np_dtype(arg.dtype)), arg.dtype.scalar) + + def create_atomic_cas(self, ptr, cmp, val, sem, scope): + if sem not in self.ir_sem_to_interpreter_sem: + raise ValueError(f"unsupported semantic {sem}") + sem = self.ir_sem_to_interpreter_sem[sem] + return TensorHandle(_interpreter.atomic_cas(ptr.data, cmp.data, val.data, sem), cmp.dtype.scalar) + + def create_atomic_rmw(self, rmwOp, ptr, val, mask, sem, scope): + if rmwOp not in self.ir_rmw_op_to_interpreter_rmw_op: + raise ValueError(f"unsupported rmwOp {rmwOp}") + if sem not in self.ir_sem_to_interpreter_sem: + raise ValueError(f"unsupported semantic {sem}") + rmwOp = self.ir_rmw_op_to_interpreter_rmw_op[rmwOp] + sem = self.ir_sem_to_interpreter_sem[sem] + return TensorHandle(_interpreter.atomic_rmw(rmwOp, ptr.data, val.data, mask.data, sem), val.dtype.scalar) + + def create_extern_elementwise(self, libName, libPath, symbol, argList, retType, isPure): + raise NotImplementedError("extern_elementwise not supported in interpreter mode") + + def create_inline_asm(self, inlineAsm, constraints, values, type, isPure, pack): + raise NotImplementedError("inline_asm not supported in interpreter mode") + + def create_print(self, prefix, hex, values, isSigned): + # NOTE: the `isSigned` variable is not really used here; because Signness is already known + # by `values` themselves in python interpreter, thus not really needed here; + # it is only used for triton PrintOpToLLVM to correctly construct the format specifier. + # Interpreter's device_print function has a different format than Triton's device_print + msg = f"({self.grid_idx[0]}, {self.grid_idx[1]}, {self.grid_idx[2]})" + if prefix: + msg += f" {prefix}" + if hex: + np.set_printoptions(formatter={'all': lambda x: f"0x{x:02x}"}) + for value in values: + print(msg + f" {value.data}") + if hex: + np.set_printoptions(formatter=None) + + def create_assert(self, condition, message): + # Interpreter's device_assert function has a different format than Triton's device_assert + assert condition, f"{message}" + + def create_assume(self, condition): + assert condition, "Assume failed" + + def create_barrier(self): + # Triton's barrier applies to each program in a grid, so it's a no-op in the interpreter + pass + + def create_make_block_ptr(self, base, shape, strides, offsets, tensor_shape, order): + # Create new offsets to avoid modifying the original + new_offsets = [offset.clone() for offset in offsets] + return BlockPointerHandle(base, shape, strides, new_offsets, tensor_shape, order) + + def create_advance(self, ptr, offsets): + if len(ptr.offsets) != len(offsets): + raise ValueError("len(ptr.offsets) != len(offsets)") + # Create new offsets to avoid modifying the original + new_offsets = [offset.clone() for offset in ptr.offsets] + ret = BlockPointerHandle(ptr.base, ptr.shape, ptr.strides, new_offsets, ptr.tensor_shape, ptr.order) + for i in range(len(offsets)): + ret.offsets[i].data += offsets[i].data + return ret + + def get_all_ones_value(self, type): + np_type = _get_np_dtype(type) + if "int" in np_type.name: + return TensorHandle(np.full(1, -1, dtype=np_type), type.scalar) + else: + raise TypeError(f"unsupported type {type}") + + +def _patch_attr(obj, name, member, builder): + new_member = lambda *args, member=member, **kwargs: (member(*args, ** + {k: v + for k, v in kwargs.items() + if k != "_builder"}, _builder=builder)) + setattr(obj, name, new_member) + + +def _patch_builtin(pkg, builder): + for name, member in inspect.getmembers(pkg): + if tl.core.is_builtin(member): + _patch_attr(pkg, name, member, builder) + + +def _patch_lang_tensor(tensor): + + def _get_bool(self): + data = self.handle.data + # in triton, only scalars can be converted to booleans + # here we need this hack because all scalars are tensors + return bool(data) if data.size == 1 else True + + def _get_transpose(self): + return tl.core.tensor(TensorHandle(np.transpose(self.handle.data), self.handle.dtype), self.dtype.scalar) + + tensor.__index__ = lambda self: int(self.handle.data) + tensor.__bool__ = lambda self: _get_bool(self) + tensor.__repr__ = lambda self: repr(self.handle.data) + tensor.__str__ = lambda self: str(self.handle.data) + tensor.T = property(_get_transpose) + + +class ReduceScanOpIneterface: + + def __init__(self, axis, combine_fn): + self.axis = axis + self.combine_fn = combine_fn + + def check_axis(self, shape, axis): + if axis is not None and axis >= len(shape): + raise ValueError(f"axis {axis} out of bounds for shape {shape}") + + def check_tensor(self, input): + for arg in input: + if not isinstance(arg, tl.core.tensor): + raise ValueError(f"input must be a tensor, got {type(arg)}") + self.check_axis(arg.shape, self.axis) + + def to_tensor(self, ret, dtype): + if hasattr(ret, "shape") and ret.shape: + ret_type = tl.block_type(dtype, ret.shape) + else: + ret = np.array([ret]).astype(_get_np_dtype(dtype)) + ret_type = dtype + return tl.core.tensor(TensorHandle(ret, dtype.scalar), ret_type) + + def apply(self, input): + if not isinstance(input, tuple): + input = (input, ) + self.check_tensor(input) + return self.apply_impl(input) + + def apply_impl(self, input): + raise NotImplementedError("apply_impl not implemented") + + +class ReduceOps(ReduceScanOpIneterface): + + def __init__(self, axis, combine_fn, keep_dims): + super().__init__(axis, combine_fn) + self.keep_dims = keep_dims + + def unravel(self, input, axis): + ret = [] + for data in input: + if axis is not None: + ret.append(data) + else: + axis = 0 + ret.append(self.to_tensor(data.handle.data.flatten(), data.dtype)) + return tuple(ret), axis + + def generic_reduce(self, input): + original_axis = self.axis + input, axis = self.unravel(input, self.axis) + input_data = [] + output_data = [] + input_shape = input[0].handle.data.shape + output_shape = input_shape[0:axis] + input_shape[axis + 1:] + for arg in input: + input_data.append(arg.handle.data) + output_data.append(np.zeros(output_shape, dtype=arg.handle.data.dtype)) + # Reduce on axis + for i in range(input_data[0].size): + # Recover input_index from i using input_shape + input_index = np.unravel_index(i, input_shape) + output_index = input_index[0:axis] + input_index[axis + 1:] + input_tuple = tuple(self.to_tensor(d[input_index], input[ii].dtype) for ii, d in enumerate(input_data)) + if input_index[axis] == 0: + # First element + for j in range(len(output_data)): + output_data[j][output_index] = input_tuple[j].handle.data.item() + else: + acc_tuple = tuple(self.to_tensor(o[output_index], input[oi].dtype) for oi, o in enumerate(output_data)) + combine_fn_ret = self.combine_fn.fn(*acc_tuple, *input_tuple) + acc_tuple = (combine_fn_ret, ) if not isinstance(combine_fn_ret, tuple) else combine_fn_ret + for j in range(len(output_data)): + output_data[j][output_index] = acc_tuple[j].handle.data.item() if isinstance( + acc_tuple[j], tl.core.tensor) else acc_tuple[j] + # Pack output + ret = [] + for i, data in enumerate(output_data): + if self.keep_dims: + if original_axis is not None: + data = np.expand_dims(data, axis) + else: + for _ in range(len(input_shape)): + data = np.expand_dims(data, 0) + + elif original_axis is None: + # Take a scalar + data = data.item() + ret.append(self.to_tensor(data, input[i].dtype)) + return ret[0] if len(ret) == 1 else tuple(ret) + + def min_max(self, input, val_reduce_op, idx_reduce_op=None): + # If input is a tuple, it must be (val, index), and we only take val + input = input[0] if isinstance(input, tuple) else input + val = None + idx = None + if val_reduce_op: + val = self.to_tensor(val_reduce_op(input.handle.data, axis=self.axis, keepdims=self.keep_dims), input.dtype) + if idx_reduce_op: + idx = self.to_tensor(idx_reduce_op(input.handle.data, axis=self.axis, keepdims=self.keep_dims), tl.int32) + if val is not None and idx is not None: + return val, idx + elif val is not None: + return val + elif idx is not None: + return idx + else: + raise ValueError("val_reduce_op and idx_reduce_op are both None") + + def sum(self, input): + return self.to_tensor(np.sum(input.handle.data, axis=self.axis, keepdims=self.keep_dims), input.dtype) + + def apply_impl(self, input): + if self.combine_fn == tl.standard._argmin_combine_tie_break_left: + return self.min_max(input[0], val_reduce_op=np.min, idx_reduce_op=np.argmin) + elif self.combine_fn == tl.standard._argmax_combine_tie_break_left: + return self.min_max(input[0], val_reduce_op=np.max, idx_reduce_op=np.argmax) + elif self.combine_fn == tl.standard._elementwise_max: + return self.min_max(input[0], val_reduce_op=np.max, idx_reduce_op=None) + elif self.combine_fn == tl.standard._elementwise_min: + return self.min_max(input[0], val_reduce_op=np.min, idx_reduce_op=None) + elif self.combine_fn == tl.standard._sum_combine: + return self.sum(input[0]) + else: + # Fall back to the slow mode + return self.generic_reduce(input) + + +class ScanOps(ReduceScanOpIneterface): + + def __init__(self, axis, combine_fn, reverse): + super().__init__(axis, combine_fn) + self.reverse = reverse + + def cumsum(self, input): + return [self.to_tensor(np.cumsum(input.handle.data, axis=self.axis), dtype=input.dtype)] + + def cumprod(self, input): + return [self.to_tensor(np.cumprod(input.handle.data, axis=self.axis), dtype=input.dtype)] + + def generic_scan(self, input): + input_data = [] + output_data = [] + shape = input[0].handle.data.shape + for arg in input: + input_data.append(arg.handle.data) + output_data.append(np.zeros(shape, dtype=arg.handle.data.dtype)) + # Scan on axis + for i in range(input_data[0].size): + # Recover index from i using shape + index = np.unravel_index(i, shape) + data = tuple(self.to_tensor(d[index], input[ii].dtype) for ii, d in enumerate(input_data)) + if index[self.axis] == 0: + # First element + for j in range(len(output_data)): + output_data[j][index] = data[j].handle.data.item() + else: + prev_index = tuple(index[i] - 1 if i == self.axis else index[i] for i in range(len(index))) + acc_tuple = tuple(self.to_tensor(o[prev_index], input[oi].dtype) for oi, o in enumerate(output_data)) + combine_fn_ret = self.combine_fn.fn(*acc_tuple, *data) + acc_tuple = (combine_fn_ret, ) if not isinstance(combine_fn_ret, tuple) else combine_fn_ret + for j in range(len(output_data)): + output_data[j][index] = acc_tuple[j].handle.data.item() if isinstance( + acc_tuple[j], tl.core.tensor) else acc_tuple[j] + # Pack output + ret = [] + for i, data in enumerate(output_data): + ret.append(self.to_tensor(data, input[i].dtype)) + return ret + + def apply_impl(self, input): + new_input = [] + if self.reverse: + for arg in input: + new_input.append(self.to_tensor(np.flip(arg.handle.data, axis=self.axis), arg.dtype)) + else: + new_input = input + if self.combine_fn == tl.standard._sum_combine: + ret = self.cumsum(new_input[0]) + elif self.combine_fn == tl.standard._prod_combine: + ret = self.cumprod(new_input[0]) + else: + # Fall back to the slow mode + ret = self.generic_scan(new_input) + if self.reverse: + for arg in ret: + arg.handle.data = np.flip(arg.handle.data, axis=self.axis) + return len(ret) == 1 and ret[0] or tuple(ret) + + +def _patch_reduce_scan(): + # Because interpreter doesn't support region_builder_fn, we cannot patch the builder + # to use the new reduce and scan functions. + # Instead, we need to patch reduce and reduce functions in tl and tl.core + def _new_reduce(input, axis, combine_fn, keep_dims=False, **kwargs): + return ReduceOps(axis, combine_fn, keep_dims).apply(input) + + def _new_scan(input, axis, combine_fn, reverse=False, **kwargs): + return ScanOps(axis, combine_fn, reverse).apply(input) + + tl.reduce = _new_reduce + tl.associative_scan = _new_scan + tl.core.reduce = _new_reduce + tl.core.associative_scan = _new_scan + + +def _patch_lang_core(lang): + + def _new_to_ir(self, builder): + # We need to specify signedness for integer types in the numpy mode + if self.name == 'void': + return builder.get_void_ty() + elif self.name == 'int1': + return builder.get_int1_ty() + elif self.name == 'int8': + return builder.get_int8_ty() + elif self.name == 'uint8': + return builder.get_uint8_ty() + elif self.name == 'int16': + return builder.get_int16_ty() + elif self.name == 'uint16': + return builder.get_uint16_ty() + elif self.name == 'int32': + return builder.get_int32_ty() + elif self.name == 'uint32': + return builder.get_uint32_ty() + elif self.name == 'int64': + return builder.get_int64_ty() + elif self.name == 'uint64': + return builder.get_uint64_ty() + elif self.name == 'fp8e5': + return builder.get_fp8e5_ty() + elif self.name == 'fp8e4nv': + return builder.get_fp8e4nv_ty() + elif self.name == 'fp8e4b15': + return builder.get_fp8e4b15_ty() + elif self.name == 'fp16': + return builder.get_half_ty() + elif self.name == 'bf16': + return builder.get_bf16_ty() + elif self.name == 'fp32': + return builder.get_float_ty() + elif self.name == 'fp64': + return builder.get_double_ty() + raise ValueError(f'fail to convert {self} to ir type') + + # can't just map lang.static_range to `range`, because `tl.static_range` + # can get `step` passed by keyword + def _new_range(arg1, arg2=None, step=None, **kwargs): + if step is None: + step = 1 + if arg2 is None: + start, end = 0, arg1 + else: + start, end = arg1, arg2 + return range(start, end, step) + + def _new_static_assert(cond, msg=""): + assert cond, msg + + def _set_attr(input, values, name): + # skip non tensor types. This may happen for induction variables. + if not isinstance(input, tl.tensor): + return input + # Unwrap constexpr + values = [values] if not isinstance(values, (list, tuple)) else values + values = [v.value if isinstance(v, tl.constexpr) else v for v in values] + if len(values) != max(1, len(input.shape)): + raise ValueError(f"len(values) != len(input.shape) for {name}") + input.handle.set_attr(name, values) + return input + + lang.range = _new_range + lang.static_range = _new_range + lang.static_assert = _new_static_assert + lang.static_print = print + lang.dtype.to_ir = _new_to_ir + lang.multiple_of = partial(_set_attr, name="tt.divisiblity") + lang.max_contiguous = partial(_set_attr, name="tt.contiguity") + lang.max_constancy = partial(_set_attr, name="tt.constancy") + + _patch_reduce_scan() + + +def _patch_lang(fn): + langs = [value for _, value in fn.__globals__.items() if value in [tl, tl.core]] + assert len(langs) >= 1, "triton.language must be visible from within jit'd function" + for lang in langs: + _patch_builtin(lang, interpreter_builder) + _patch_builtin(lang.tensor, interpreter_builder) + if lang == tl: + _patch_builtin(lang.math, interpreter_builder) + _patch_lang_tensor(lang.tensor) + _patch_lang_core(lang) + + +# TODO: wrap everything in triton tensors +def _implicit_cvt(arg): + if isinstance(arg, int): + ty = tl.str_to_ty(triton.runtime.jit.JITFunction._type_of(triton.runtime.jit.JITFunction._key_of(arg))) + dtype = np.int32 + if -2**31 <= arg < 2**31: + dtype = np.int32 + elif 2**31 <= arg < 2**32: + dtype = np.uint32 + elif -2**63 <= arg < 2**63: + dtype = np.int64 + elif 2**63 <= arg < 2**64: + dtype = np.uint64 + else: + raise ValueError(f"Unsupported integer value {arg}") + handle = TensorHandle(np.array([arg], dtype=dtype), ty) + return tl.tensor(handle, ty) + if hasattr(arg, "data_ptr"): + ty = tl.str_to_ty(triton.runtime.jit.JITFunction._type_of(triton.runtime.jit.JITFunction._key_of(arg))) + handle = TensorHandle(np.array([arg.data_ptr()], dtype=np.uint64), ty) + return tl.tensor(handle, ty) + return arg + + +interpreter_builder = InterpreterBuilder() + +# These keywords are not supported by the interpreter +RESERVED_KWS = ["num_warps", "num_stages", "num_ctas", "enable_fp_fusion", "grid", "maxnreg"] + + +class GridExecutor: + + def __init__(self, fn, arg_names, grid): + from .jit import _normalize_ty # TODO: modularize + + self.fn = fn + self.arg_names = arg_names + self.grid = grid + __annotations__ = {name: _normalize_ty(ty) for name, ty in fn.__annotations__.items()} + self.constexprs = [name for name in arg_names if __annotations__.get(name) == "constexpr"] + + def _init_args_hst(self, args_dev, kwargs): + args_hst = [] + for arg in args_dev: + if hasattr(arg, "data_ptr"): + args_hst.append(arg.cpu()) + else: + args_hst.append(arg) + # Process keyword arguments + kwargs_hst = {} + for key, value in kwargs.items(): + if hasattr(value, "data_ptr"): + kwargs_hst[key] = value.cpu() + else: + kwargs_hst[key] = value + return args_hst, kwargs_hst + + def _restore_args_dev(self, args_dev, args_hst, kwargs, kwargs_hst): + for arg_dev, arg_hst in zip(args_dev, args_hst): + if hasattr(arg_dev, "data_ptr"): + arg_dev.data.copy_(arg_hst.to(arg_dev.device).data) + + # Restore keyword arguments + for key, kwarg_dev in kwargs.items(): + kwarg_hst = kwargs_hst[key] + if hasattr(kwarg_dev, "data_ptr"): + kwarg_dev.data.copy_(kwarg_hst.to(kwarg_dev.device).data) + + def __call__(self, *args_dev, **kwargs): + # removes reserved keywords from kwargs + kwargs = {k: v for k, v in kwargs.items() if k not in RESERVED_KWS} + if kwargs.pop("warmup", False): + return + # copy arguments to the host + args_hst, kwargs_hst = self._init_args_hst(args_dev, kwargs) + # remaps core language functions to interpreted ones + _patch_lang(self.fn) + # we need to copy arguments to the host for the interpreter + # implicitly convert tensor arguments to their base pointers + args = inspect.getcallargs(self.fn, *args_hst, **kwargs_hst) + args = {name: arg if name in self.constexprs else _implicit_cvt(arg) for name, arg in args.items()} + # iterate through grid + grid = self.grid(args) if callable(self.grid) else self.grid + assert len(grid) <= 3, "grid must have at most 3 dimensions" + grid = grid + (1, ) * (3 - len(grid)) + interpreter_builder.set_grid_dim(*grid) + try: + for x in range(grid[0]): + for y in range(grid[1]): + for z in range(grid[2]): + interpreter_builder.set_grid_idx(x, y, z) + self.fn(**args) + except Exception as e: + raise InterpreterError(repr(e)) from e + # copy arguments back to propagate side-effects + self._restore_args_dev(args_dev, args_hst, kwargs, kwargs_hst) + + +class ASTTransformer(ast.NodeTransformer): + + def visit_Assign(self, node): + names = [] + for target in node.targets: + names += [self.visit(target)] + if len(names) > 1: + raise ValueError("Multiple assignments are not supported") + # Modify the assignment x = value to + # triton.language.semantic.to_tensor(value, interpreter_builder, False) + node.value = ast.Call( + func=ast.Attribute( + value=ast.Attribute( + value=ast.Attribute(value=ast.Name(id='triton', ctx=ast.Load()), attr='language', ctx=ast.Load()), + attr='semantic', ctx=ast.Load()), attr='to_tensor', ctx=ast.Load()), + args=[node.value, ast.Name(id='interpreter_builder', ctx=ast.Load()), + ast.Constant(value=False)], keywords=[]) + return node + + +class FunctionRewriter: + ast_transformer = ASTTransformer() + + def __init__(self, fn, **kwargs): + self.fn = fn + self.kwargs = kwargs + self.filename: str = "" + # Absolute line number in the file + self.def_file_lineno: int = 0 + + def rewrite_ast(self): + # If exception is raise, it means the function does not have source code available, + # e.g., dynamically generated functions, we cannot rewrite it so just return the original function + try: + lines, _ = inspect.getsourcelines(self.fn) + except Exception: + return self.fn + + # truncate lines before def + # @triton.autotune(...) + # ... + # @triton.jit + # ... + # def foo(...): <- this line is the function definition + self.filename, self.def_file_lineno = self._get_jit_fn_file_line() + self.def_lineno = self._find_def(lines) + src = self._prepare_source(lines) + transformed_ast = self._transform_ast(src) + return self._compile_and_exec(transformed_ast) + + def _get_jit_fn_file_line(self): + from .jit import get_jit_fn_file_line, JITFunction + return get_jit_fn_file_line(JITFunction(self.fn)) + + def _find_def(self, lines): + def_lineno = 0 + # Line numbers start from 1 + for i, line in enumerate(lines): + if line.strip().startswith("def "): + def_lineno = i + 1 + return def_lineno + + def _prepare_source(self, lines): + lines = lines[self.def_lineno - 1:] + src = ''.join(lines) + return textwrap.dedent(src) + + def _transform_ast(self, src): + # src is like: + # 1: def foo(...): + # 2: ... + parsed_ast = ast.parse(src) + transformed_ast = self.ast_transformer.visit(parsed_ast) + ast.fix_missing_locations(transformed_ast) + inc_lineno = self.def_file_lineno - 1 + ast.increment_lineno(transformed_ast, inc_lineno) + return transformed_ast + + def _compile_and_exec(self, transformed_ast): + compiled_code = compile(transformed_ast, filename=self.filename, mode='exec') + local_namespace = {**self.kwargs} + fn_globals = self.fn.__globals__ + for key, value in globals().items(): + if key not in fn_globals: + fn_globals[key] = value + exec(compiled_code, fn_globals, local_namespace) + return local_namespace[self.fn.__name__] + + +class InterpretedFunction: + # Cache all rewritten functions + rewritten_fn = {} + + def __init__(self, fn, **kwargs) -> None: + self.fn = fn + self.rewriter = FunctionRewriter(fn, **kwargs) + + def run(*args, **kwargs): + grid = kwargs["grid"] + fn = self.rewrite() + return GridExecutor(fn, self.arg_names, grid)(*args, **kwargs) + + self.run = run + signature = inspect.signature(fn) + self.arg_names = [v.name for v in signature.parameters.values()] + + def rewrite(self): + if self.fn not in self.rewritten_fn: + self.rewritten_fn[self.fn] = self.rewriter.rewrite_ast() + return self.rewritten_fn[self.fn] + + @property + def __name__(self): + return self.fn.__name__ + + def __getitem__(self, grid): + fn = self.rewrite() + return GridExecutor(fn, self.arg_names, grid) + + def __call__(self, *args, **kwargs): + # This is a device function call + _patch_lang(self.fn) + fn = self.rewrite() + try: + return fn(*args, **kwargs) + except Exception as e: + raise InterpreterError(repr(e)) from e diff --git a/lib/python3.12/site-packages/triton/runtime/jit.py b/lib/python3.12/site-packages/triton/runtime/jit.py new file mode 100644 index 0000000000000000000000000000000000000000..45178a40bb29069f9f7c7f33a23df54ce891433d --- /dev/null +++ b/lib/python3.12/site-packages/triton/runtime/jit.py @@ -0,0 +1,951 @@ +from __future__ import annotations, division +import ast +import hashlib +import inspect +import itertools +import os +import re +import textwrap +from collections import defaultdict +from functools import cached_property +from typing import Callable, Generic, Iterable, Optional, TypeVar, Union, overload, Dict, Any, Tuple +from ..runtime.driver import driver +from types import ModuleType + +TRITON_MODULE = __name__[:-len(".runtime.jit")] + +T = TypeVar("T") + +# ----------------------------------------------------------------------------- +# Dependencies Finder +# ----------------------------------------------------------------------------- + + +class DependenciesFinder(ast.NodeVisitor): + """ + This AST visitor is used to find dependencies of a JITFunction. This can + be used to invalidate a JITFunction's hash when its source code -- or + that of its dependencies -- changes. + + This visitor also keeps track of the global variables touched by the + JITFunction. When we launch the kernel, we check that these have the same + values as they did when we ran this visitor. If not, we raise an error (or + otherwise we could recompile). + """ + + def __init__(self, name, globals, src) -> None: + super().__init__() + self.name = name + self.hasher = hashlib.sha256(src.encode("utf-8")) + + # This function's __globals__ dict. + self.globals = globals + + # Python builtins that can be accessed from Triton kernels. + self.supported_python_builtins = { + 'float', + 'getattr', + 'int', + 'isinstance', + 'len', + 'list', + 'max', + 'min', + 'print', + 'range', + } + + # used_global_vals tells us which global variables are used by this + # function and all those it transitively calls, plus the values of those + # variables when each function was initially run. (That is, if A calls + # C, and B calls C, then the values for C in used_global_vals will be + # from the first time C was run, either by A or B.) + # + # Each function may have a different __globals__ dict, so the global + # variable `foo` may actually have a different value in the different + # functions. Thus this map is actually + # (var_name, id(__globals__)) -> (var_value, __globals__). + self.used_global_vals: Dict[Tuple[str, int], Tuple[Any, Dict[str, Any]]] = {} + + self.visiting_arg_default_value = False + + @property + def ret(self): + return self.hasher.hexdigest() + + def _is_triton_builtin(self, node, func): + if inspect.isbuiltin(node.func): + return True + module = getattr(func, "__module__", "") + return module.startswith(TRITON_MODULE) + + def _update_hash(self, func): + if isinstance(func, JITFunction): + # Merge our used_global_vals with those of the called function, + # after checking that all overlapping values are consistent. + for k in self.used_global_vals.keys() & func.used_global_vals.keys(): + var_name, _ = k + v1, _ = self.used_global_vals[k] + v2, _ = func.used_global_vals[k] + if v1 != v2: + raise RuntimeError( + f"Global variable {var_name} has value {v1} when compiling {self.name}, but inner kernel {func.__name__} has conflicting value {v2} from when it was first compiled. This is not allowed." + ) + self.used_global_vals.update(func.used_global_vals) + # update hash + func_key = func.cache_key + func_key += str(getattr(func, "noinline", False)) + self.hasher.update(func_key.encode("utf-8")) + + def visit_Name(self, node): + if type(node.ctx) is ast.Store: + return node.id + + if node.id in self.local_names: + # The global name is hidden by the local name. + return None + + val = self.globals.get(node.id, None) + + # Only keep track of "interesting" global variables, that non-evil users + # might change. Don't consider functions, modules, builtins, etc. This + # helps keep the list of vars we have to check small. + if (val is not None # + # Python default arguments are resolved only once, when the + # function is defined. So if you do `foo(a=A)` and the value of + # A changes, foo will still use the old value of A. + and not self.visiting_arg_default_value + # It would be pretty evil if someone did `import x` and then + # `x = blah`. + and type(val) is not ModuleType + # It would be pretty evil if we used function `foo` inside of + # `bar` and then someone did `foo = baz`. + and not isinstance(val, JITFunction) and not getattr(val, "__triton_builtin__", False) # + and node.id not in self.supported_python_builtins): + self.used_global_vals[(node.id, id(self.globals))] = (val, self.globals) + + self._update_hash(val) + return val + + def visit_Tuple(self, node): + # We need to explicitly return the tuple values so that visit_Assign can + # access them in the case of `a, b = ...`. + return [self.visit(elt) for elt in node.elts] + + def visit_Attribute(self, node): + lhs = self.visit(node.value) + while isinstance(lhs, ast.Attribute): + lhs = self.visit(lhs.value) + if lhs is None or (getattr(lhs, "__name__", "") == TRITON_MODULE): + return None + ret = getattr(lhs, node.attr) + self._update_hash(ret) + return ret + + def visit_FunctionDef(self, node): + # Save the local name, which may hide the global name. + self.local_names = {arg.arg for arg in node.args.args} + self.generic_visit(node) + + def visit_arguments(self, node): + # The purpose of this function is to visit everything in `arguments` + # just like `generic_visit`, except when we're visiting default values + # (i.e. the `foo` part of `def fn(x = foo)`), we set + # self.visiting_arg_default_value = True. This allows visit_Name to be + # aware that we're inside function default values, which have special + # semantics. + + # According to the AST docs, the arguments node has the following structure. + # + # arguments = (arg* posonlyargs, arg* args, arg? vararg, arg* kwonlyargs, + # expr* kw_defaults, arg? kwarg, expr* defaults) + def visit_defaults(defaults): + try: + assert not self.visiting_arg_default_value + self.visiting_arg_default_value = True + for expr in defaults: + if expr is not None: + self.visit(expr) + finally: + self.visiting_arg_default_value = False + + for arg in itertools.chain(node.posonlyargs, node.args, [node.vararg] if node.vararg else [], node.kwonlyargs): + self.visit(arg) + + visit_defaults(node.kw_defaults) + + if node.kwarg is not None: + self.visit(node.kwarg) + + visit_defaults(node.defaults) + + def visitAssnTarget(self, node): + # Target is either a single string, or a list of strings (if the assn + # target is a tuple). + target = self.visit(node) + if isinstance(target, list): + self.local_names |= set(target) + else: + self.local_names.add(target) + + def visit_Assign(self, node): + if len(node.targets) != 1: + # TODO(jlebar): I don't actually know how to hit this. You don't + # get it from `a, b = ...` -- in that case, node.targets is a single + # Tuple, and in fact we *do* need to handle that case if we want + # existing code to work. + raise TypeError("Simultaneous multiple assignment is not supported.") + + self.visitAssnTarget(node.targets[0]) + + # This will re-visit the target, but that's OK. + self.generic_visit(node) + + def visit_AnnAssign(self, node): + self.visitAssnTarget(node.target) + + # This will re-visit the target, but that's OK. + self.generic_visit(node) + + def visit_For(self, node): + self.visitAssnTarget(node.target) + + # This will re-visit the target, but that's fine. + self.generic_visit(node) + + +# ----------------------------------------------------------------------------- +# JITFunction +# ----------------------------------------------------------------------------- + + +def _normalize_ty(ty) -> str: + if isinstance(ty, type): + return ty.__name__ + elif isinstance(ty, str): + return ty + return repr(ty) + + +class KernelParam: + """Represents a parameter (name plus metadata) to a @jit'ed function.""" + + def __init__(self, num: int, param: inspect.Parameter, do_not_specialize: bool, + do_not_specialize_on_alignment: bool): + self.num = num + self._param = param + self.do_not_specialize = do_not_specialize + self.do_not_specialize_on_alignment = do_not_specialize_on_alignment + + @cached_property + def name(self): + return self._param.name + + @cached_property + def annotation(self): + if not self._param.annotation or self._param.annotation == inspect.Parameter.empty: + return "" + return _normalize_ty(self._param.annotation) + + @cached_property + def annotation_type(self): + annotation = self.annotation + for ty1, ty2 in [("uint", 'u'), ("int", 'i')]: + width = annotation[annotation.find(ty1) + len(ty1):] + if width and ty1 in annotation: + return f"{ty2}{width}" + if annotation == "bool": + return "u1" + return "" + + @cached_property + def is_constexpr(self): + return "constexpr" in self.annotation + + @cached_property + def is_const(self): + return "const" in self.annotation and not self.is_constexpr + + @property + def default(self): + return self._param.default + + @property + def has_default(self): + return self._param.default != inspect.Parameter.empty + + +def compute_spec_key(v, align): + + if align and hasattr(v, "data_ptr") and (v.data_ptr() % 16 == 0): + return "D" + elif isinstance(v, int): + # bool is a subclass of int, so we don't check explicitly above. + if align and (v % 16 == 0): + return "D" + elif v == 1: + return "1" + return "N" + + +dtype2str = {} + + +def mangle_type(arg, is_const=False): + + if arg is None: + return "none" + elif isinstance(arg, bool): + return "i1" + elif isinstance(arg, int): + if -(2**31) <= arg and arg <= 2**31 - 1: + return "i32" + elif 2**63 <= arg and arg <= 2**64 - 1: + return "u64" + else: + return "i64" + elif isinstance(arg, float): + return "fp32" + elif hasattr(arg, "tma_desc_cpu_ptr"): + return "nvTmaDesc" + else: + # dtypes are hashable so we can memoize this mapping: + dsk = (arg.dtype, is_const) + res = dtype2str.get(dsk, None) + if res is None: + res = ("*k" if dsk[1] else "*") + type_canonicalisation_dict[str(dsk[0]).split('.')[-1]] + dtype2str[dsk] = res + return res + + +class KernelInterface(Generic[T]): + run: T + + def __getitem__(self, grid) -> T: + """ + A JIT function is launched with: fn[grid](*args, **kwargs). + Hence JITFunction.__getitem__ returns a callable proxy that + memorizes the grid. + """ + return lambda *args, **kwargs: self.run(grid=grid, warmup=False, *args, **kwargs) + # return cast(T, functools.partial(cast(Callable, self.run), grid=grid)) + + +def serialize_specialization_data(name, signature, constants, attrs, options, key): + constants = {key: str(value) if value.__class__.__name__ == "dtype" else value for key, value in constants.items()} + import json + obj = { + 'name': name, 'signature': signature, 'constants': constants, 'attrs': attrs.to_dict(), 'options': + options.__dict__, 'key': key + } + serialized_obj = json.dumps(obj) + return serialized_obj + + +def create_function_from_signature(sig, kparams, backend): + """ + Equivalent to sig.bind followed by apply_defaults. This generates a + native Python function (using exec) which can be memoized on a per-kernel + basis to avoid having to run these expensive functions -- which constitute + much of the kernel launch overhead -- every time we run the kernel. + """ + + assert len(sig.parameters) == len(kparams) + + # Create the function argument list and the dict entries for the return statement + func_args = [] + dict_entries = [] + constexpr_vals = [] + non_constexpr_vals = [] + signature_types = [] + specialisations = [] + + for ((name, sp), kp) in zip(sig.parameters.items(), kparams): + if sp.default is inspect.Parameter.empty: + func_args.append(name) + dict_entries.append(f"'{name}': {name}") + else: + func_args.append(f"{name}=default_{name}") + dict_entries.append(f"'{name}': {name}") + if kp.is_constexpr: + constexpr_vals.append(name) + else: + non_constexpr_vals.append(name) + if not kp.do_not_specialize: + if not kp.do_not_specialize_on_alignment: + specialisations.append('compute_spec_key(%s, align=True)' % name) + else: + specialisations.append('compute_spec_key(%s, align=False)' % name) + if kp.annotation_type: + signature_types.append('"%s"' % kp.annotation_type) + else: + signature_types.append('mangle_type(%s, %s)' % (name, 'True' if kp.is_const else 'False')) + + cache_key = ''.join([x + ', ' for x in signature_types + specialisations]) + constexpr_vals = ''.join([x + ', ' for x in constexpr_vals]) + non_constexpr_vals = ''.join([x + ', ' for x in non_constexpr_vals]) + + func_args.append('**excess_kwargs') + + # Join all arguments into a function definition string + args_str = ', '.join(func_args) + dict_str = ', '.join(dict_entries) + func_body = "def dynamic_func(%s):\n return {%s}, (%s), (%s), (%s), excess_kwargs" % ( + args_str, dict_str, cache_key, constexpr_vals, non_constexpr_vals) + + # Prepare defaults to be inserted into function namespace + func_namespace = { + f"default_{name}": param.default + for name, param in sig.parameters.items() + if param.default is not inspect.Parameter.empty + } + + func_namespace['mangle_type'] = mangle_type + func_namespace['compute_spec_key'] = backend.compute_spec_key + + # Execute the function string in func_namespace to create the function + exec(func_body, func_namespace) + + # Extract the newly created function from the namespace + return func_namespace['dynamic_func'] + + +type_canonicalisation_dict = { + "bool": "i1", + "float8e4nv": "fp8e4nv", + "float8e5": "fp8e5", + "float8e4b15": "fp8e4b15", + "float8_e4m3fn": "fp8e4nv", + "float8e4b8": "fp8e4b8", + "float8_e4m3fnuz": "fp8e4b8", + "float8_e5m2": "fp8e5", + "float8e5b16": "fp8e5b16", + "float8_e5m2fnuz": "fp8e5b16", + "float16": "fp16", + "bfloat16": "bf16", + "float32": "fp32", + "float64": "fp64", + "int8": "i8", + "int16": "i16", + "int32": "i32", + "int64": "i64", + "uint8": "u8", + "uint16": "u16", + "uint32": "u32", + "uint64": "u64", +} + +for v in list(type_canonicalisation_dict.values()): + type_canonicalisation_dict[v] = v + + +class JITFunction(KernelInterface[T]): + # Hook for inspecting compiled functions and modules + cache_hook = None + # Hook to signal that a kernel is done compiling and inspect compiled function. + # cache_hook will always be called before compilation and compiled_hook after. + compiled_hook = None + + @staticmethod + def _key_of(arg): + if hasattr(arg, "dtype"): + return arg.dtype + elif isinstance(arg, bool): + return "i1" + elif isinstance(arg, int): + if -(2**31) <= arg and arg <= 2**31 - 1: + return "i32" + elif 2**63 <= arg and arg <= 2**64 - 1: + return "u64" + else: + return "i64" + elif isinstance(arg, float): + return "fp32" + elif arg is None: + return None + else: + raise TypeError(f"Unsupported type {type(arg)} for {arg}") + + @staticmethod + def _type_of(key, is_const=False): + # `None` is nullptr. Implicitly convert to *i8. + if key is None: + return "*i8" + elif isinstance(key, str): + return key + + dtype_str = str(key).split(".")[-1] + dtype_str = type_canonicalisation_dict[dtype_str] + const_str = "*k" if is_const else "*" + return const_str + dtype_str + + def _make_constants(self, constexpr_key): + constants = dict(zip(self.constexprs, constexpr_key)) + return constants + + def _call_hook( + self, + key, + signature, + device, + constants, + options, + configs, + is_warmup, + before, + ): + hook = JITFunction.cache_hook if before else JITFunction.compiled_hook + if hook is None: + return False + + name = self.fn.__name__ + module = self.fn.__module__ + arg_reprs = ", ".join([f"{param.name}: {ty}" for param, ty in zip(self.params, key[1])]) + repr = f"{name}[num_warps={options.num_warps}, num_ctas={options.num_ctas}, num_stages={options.num_stages}, enable_fp_fusion={options.enable_fp_fusion}]({arg_reprs})" + + class JitFunctionInfo: + + def __init__(self, module, name, jit_function): + self.module = module + self.name = name + self.jit_function = jit_function + pass + + specialization_data = serialize_specialization_data(name, signature, constants, configs[0], options, key) + + kwargs = { + 'signature': signature, + 'device': device, + 'constants': constants, + 'num_warps': options.num_warps, + 'num_ctas': options.num_ctas, + 'num_stages': options.num_stages, + 'enable_fp_fusion': options.enable_fp_fusion, + 'extern_libs': options.extern_libs, + 'configs': configs, + 'specialization_data': specialization_data, + 'is_warmup': is_warmup, + } + + return hook( + key=key, + repr=repr, + fn=JitFunctionInfo(module, name, self), + compile={"key": key, **kwargs}, + is_manual_warmup=is_warmup, + already_compiled=False, + ) + + def add_pre_run_hook(self, hook): + ''' + Add a hook that will be executed prior to the execution of run + function with args and kwargs passed into the kernel + ''' + assert callable(hook) + self.pre_run_hooks.append(hook) + + def create_binder(self, backend): + """ + Precompute as much as possible. + """ + from ..compiler import CompiledKernel, compile, ASTSource, make_backend + self.CompiledKernel = CompiledKernel + self.compile = compile + self.ASTSource = ASTSource + self.make_backend = make_backend + self.binder = create_function_from_signature(self.signature, self.params, backend) + self.constexpr_indices = [i for (i, p) in enumerate(self.params) if p.is_constexpr] + self.non_constexpr_indices = [i for (i, p) in enumerate(self.params) if not p.is_constexpr] + self.specialised_indices = [ + i for (i, p) in enumerate(self.params) if (not p.do_not_specialize) and (not p.is_constexpr) + ] + + def run(self, *args, grid, warmup, **kwargs): + kwargs["debug"] = kwargs.get("debug", False) or os.environ.get("TRITON_DEBUG", "0") == "1" + + # parse options + from ..compiler import make_backend + device = driver.active.get_current_device() + stream = driver.active.get_current_stream(device) + target = driver.active.get_current_target() + backend = make_backend(target) + + # Execute pre run hooks with args and kwargs + for hook in self.pre_run_hooks: + hook(*args, **kwargs) + + if self.binder is None: + self.create_binder(backend) + + bound_args, sig_and_spec, constexpr_vals, non_constexpr_vals, excess_kwargs = self.binder(*args, **kwargs) + + # compute cache key + key = ''.join(sig_and_spec) + str((constexpr_vals, excess_kwargs)) + kernel = self.cache[device].get(key, None) + + if kernel is None: + # Kernel is not cached; we have to compile. + options = backend.parse_options(kwargs) + + # deprecated arguments + assert "device_type" not in kwargs, "device_type option is deprecated; current target will be used" + assert "device" not in kwargs, "device option is deprecated; current device will be used" + assert "stream" not in kwargs, "stream option is deprecated; current stream will be used" + for k in excess_kwargs: + if k not in options.__dict__: + raise KeyError("Keyword argument %s was specified but unrecognised" % k) + + bound_vals = tuple(bound_args.values()) + + # `None` is nullptr. Implicitly convert to *i8. This needs to be + # done here rather than when we build the signature as otherwise + # the kernel cache key could not distinguish between byte pointers + # and None arguments, resulting in a downstream mismatch: + sigkeys = [self.params[i].name for i in self.non_constexpr_indices] + sigvals = sig_and_spec[:len(sigkeys)] + signature = {k: ('*i8' if (v == 'none') else v) for (k, v) in zip(sigkeys, sigvals)} + + configs = (backend.get_attrs_descriptor(self.params, bound_vals), ) + constant_params = configs[0].get_constants() + constants = { + p.name: v + for (v, p) in zip(bound_vals, self.params) + if p.is_constexpr or (p.num in constant_params) or v is None + } + for i, arg in constants.items(): + if callable(arg): + raise TypeError(f"Callable constexpr at index {i} is not supported") + + if self._call_hook(key, signature, device, constants, options, configs, warmup, before=True): + return None + # compile the kernel + src = self.ASTSource(self, signature, constants, configs[0]) + kernel = self.compile( + src, + target=target, + options=options.__dict__, + ) + self.cache[device][key] = kernel + self._call_hook(key, signature, device, constants, options, configs, warmup, before=False) + + # Check that used global values have not changed. + not_present = object() + for (name, _), (val, globals_dict) in self.used_global_vals.items(): + if (newVal := globals_dict.get(name, not_present)) != val: + raise RuntimeError( + f"Global variable {name} has changed since we compiled this kernel, from {val} to {newVal}") + + if not warmup: + # canonicalize grid + assert grid is not None + if callable(grid): + # Arguments are passed as a dict to `grid`, by contract. + # TODO(jlebar): In the new launch API, pass the compiler flags as a + # second parameter to `grid`. + grid = grid(bound_args) + grid_size = len(grid) + grid_0 = grid[0] + grid_1 = grid[1] if grid_size > 1 else 1 + grid_2 = grid[2] if grid_size > 2 else 1 + + # launch kernel + launch_metadata = kernel.launch_metadata(grid, stream, *non_constexpr_vals) + kernel.run(grid_0, grid_1, grid_2, stream, kernel.function, kernel.packed_metadata, launch_metadata, + self.CompiledKernel.launch_enter_hook, self.CompiledKernel.launch_exit_hook, *non_constexpr_vals) + return kernel + + def __init__(self, fn, version=None, do_not_specialize=None, do_not_specialize_on_alignment=None, debug=None, + noinline=None, repr=None, launch_metadata=None): + do_not_specialize = do_not_specialize if do_not_specialize else [] + do_not_specialize_on_alignment = do_not_specialize_on_alignment if do_not_specialize_on_alignment else [] + + self.fn = fn + self.module = fn.__module__ + self.version = version + self.signature = inspect.signature(fn) + self.do_not_specialize = do_not_specialize + self.do_not_specialize_on_alignment = do_not_specialize_on_alignment + self.starting_line_number = inspect.getsourcelines(fn)[1] + self.repr = lambda _: fn.__name__ if repr is None else repr(_) + self.launch_metadata = launch_metadata + + self.binder = None + + self.params = [] + for i, param in enumerate(self.signature.parameters.values()): + dns = i in do_not_specialize or param.name in do_not_specialize + dns_oa = i in do_not_specialize_on_alignment or param.name in do_not_specialize_on_alignment + self.params.append(KernelParam(i, param, dns, dns_oa)) + + # function source code (without decorators) + self.src = textwrap.dedent(inspect.getsource(fn)) + self.src = self.src[re.search(r"^def\s+\w+\s*\(", self.src, re.MULTILINE).start():] + # cache of just-in-time compiled kernels + self.cache = defaultdict(dict) + self.hash = None + + # Map of global variables used by the function and any functions it + # transitively calls, plus their values. The values are collected when + # the function is first compiled. Then every time we run the function, + # we check that the values of the globals match what's expected, + # otherwise we raise an error. + # + # Different functions can have different __globals__ maps, so the map + # key is actually (var name, id(__globals__)), and the map value is + # (value, __globals__). + self.used_global_vals: Dict[Tuple[str, int], Tuple[Any, Dict[str, Any]]] = {} + + # JITFunction can be instantiated as kernel + # when called with a grid using __getitem__ + self.kernel = None + self.noinline = noinline + + # TODO(jlebar): Remove uses of these fields outside this file, then + # remove the fields here. + self.arg_names = [p.name for p in self.params] + self.constexprs = [p.num for p in self.params if p.is_constexpr] + + # Hooks that will be called prior to executing "run" + self.pre_run_hooks = [] + + # reuse docs of wrapped function + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + self.__globals__ = fn.__globals__ + self.__module__ = fn.__module__ + + @property + def cache_key(self): + # TODO : hash should be attribute of `self` + if self.hash is None: + dependencies_finder = DependenciesFinder(name=self.__name__, globals=self.__globals__, src=self.src) + dependencies_finder.visit(self.parse()) + self.hash = dependencies_finder.ret + str(self.starting_line_number) + self.used_global_vals = dict(sorted(dependencies_finder.used_global_vals.items())) + return self.hash + + def warmup(self, *args, grid, **kwargs): + return self.run(grid=grid, warmup=True, *map(MockTensor.wrap_dtype, args), **kwargs) + + def preload(self, specialization_data): + from ..compiler import compile, ASTSource + from triton.backends.compiler import AttrsDescriptor + import json + import triton.language as tl + device = driver.active.get_current_device() + deserialized_obj = json.loads(specialization_data) + if deserialized_obj['name'] != self.fn.__name__: + raise RuntimeError( + f"Specialization data is for {deserialized_obj['name']} but trying to preload for {self.fn.__name__}") + constants = { + key: tl.dtype(value) if tl.dtype.is_dtype(value) else value + for key, value in deserialized_obj['constants'].items() + } + signature = dict(deserialized_obj['signature'].items()) + src = ASTSource(self, signature, constants, AttrsDescriptor.from_dict(deserialized_obj['attrs'])) + options = { + key: tuple(value) if isinstance(value, list) else value + for key, value in deserialized_obj['options'].items() + } + key = deserialized_obj['key'] + kernel = compile(src, None, options) + self.cache[device][key] = kernel + return kernel + + # we do not parse `src` in the constructor because + # the user might want to monkey-patch self.src dynamically. + # Our unit tests do this, for example. + def parse(self): + tree = ast.parse(self.src) + assert isinstance(tree, ast.Module) + assert len(tree.body) == 1 + assert isinstance(tree.body[0], ast.FunctionDef) + return tree + + def __call__(self, *args, **kwargs): + raise RuntimeError("Cannot call @triton.jit'd outside of the scope of a kernel") + + def __setattr__(self, name, value): + super(JITFunction, self).__setattr__(name, value) + # - when `.src` attribute is set, cache path needs + # to be reinitialized + if name == "src": + self.hash = None + + def __repr__(self): + return f"JITFunction({self.module}:{self.fn.__name__})" + + +# ----------------------------------------------------------------------------- +# `jit` decorator +# ----------------------------------------------------------------------------- + + +@overload +def jit(fn: T) -> JITFunction[T]: + ... + + +@overload +def jit( + *, + version=None, + repr: Optional[Callable] = None, + launch_metadata: Optional[Callable] = None, + do_not_specialize: Optional[Iterable[int]] = None, + do_not_specialize_on_alignment: Optional[Iterable[int]] = None, + debug: Optional[bool] = None, + noinline: Optional[bool] = None, +) -> Callable[[T], JITFunction[T]]: + ... + + +def jit( + fn: Optional[T] = None, + *, + version=None, + repr: Optional[Callable] = None, + launch_metadata: Optional[Callable] = None, + do_not_specialize: Optional[Iterable[int]] = None, + do_not_specialize_on_alignment: Optional[Iterable[int]] = None, + debug: Optional[bool] = None, + noinline: Optional[bool] = None, +) -> Union[JITFunction[T], Callable[[T], JITFunction[T]]]: + """ + Decorator for JIT-compiling a function using the Triton compiler. + + :note: When a jit'd function is called, arguments are + implicitly converted to pointers if they have a :code:`.data_ptr()` method + and a `.dtype` attribute. + + :note: This function will be compiled and run on the GPU. It will only have access to: + + * python primitives, + * builtins within the triton package, + * arguments to this function, + * other jit'd functions + + :param fn: the function to be jit-compiled + :type fn: Callable + """ + + def decorator(fn: T) -> JITFunction[T]: + assert callable(fn) + if os.getenv("TRITON_INTERPRET", "0") == "1": + from .interpreter import InterpretedFunction + return InterpretedFunction(fn, version=version, do_not_specialize=do_not_specialize, + do_not_specialize_on_alignment=do_not_specialize_on_alignment, debug=debug, + noinline=noinline, repr=repr, launch_metadata=launch_metadata) + else: + return JITFunction( + fn, + version=version, + do_not_specialize=do_not_specialize, + do_not_specialize_on_alignment=do_not_specialize_on_alignment, + debug=debug, + noinline=noinline, + repr=repr, + launch_metadata=launch_metadata, + ) + + if fn is not None: + return decorator(fn) + + else: + return decorator + + +# ----------------------------------------------------------------------------- +# Utilities for mocking tensors +# ----------------------------------------------------------------------------- + + +class MockTensor: + """ + Can be used in place of real tensors when calling: + kernel.warmup(MockTensor(torch.float32), ...) + """ + + @staticmethod + def wrap_dtype(arg): + if arg.__class__.__name__ == "dtype" and arg.__module__ == "torch": + return MockTensor(arg) + return arg + + def __init__(self, dtype): + self.dtype = dtype + + @staticmethod + def data_ptr(): + return 0 # optimistically assumes multiple of 16 + + @staticmethod + def ptr_range(): + return 0 # optimistically assumes 32 bit pointer range + + +class TensorWrapper: + + def __init__(self, base, dtype): + self.dtype = dtype + self.base = base + self.data = base.data + self.device = base.device + self.shape = self.base.shape + + def data_ptr(self): + return self.base.data_ptr() + + def stride(self, i): + return self.base.stride(i) + + def __str__(self) -> str: + return f"TensorWrapper[{self.dtype}]({self.base})" + + def element_size(self): + return self.base.element_size() + + def cpu(self): + return TensorWrapper(self.base.cpu(), self.dtype) + + def copy_(self, other): + self.base.copy_(other.base) + + def clone(self): + return TensorWrapper(self.base.clone(), self.dtype) + + def to(self, device): + return TensorWrapper(self.base.to(device), self.dtype) + + +def reinterpret(tensor, dtype): + if isinstance(tensor, TensorWrapper): + if dtype == tensor.base.dtype: + # Reinterpreting to the original interpretation; return the base. + return tensor.base + else: + # Reinterpreting a wrapped tensor to a different type. + return TensorWrapper(tensor.base, dtype) + elif hasattr(tensor, "data_ptr"): + # A new wrapper is needed around an unwrapped tensor. + return TensorWrapper(tensor, dtype) + else: + raise TypeError(f"Cannot reinterpret a {type(tensor)}.") + + +def get_jit_fn_file_line(fn): + base_fn = fn + while not isinstance(base_fn, JITFunction): + base_fn = base_fn.fn + file_name = base_fn.fn.__code__.co_filename + lines, begin_line = inspect.getsourcelines(base_fn.fn) + # Match the following pattern: + # @triton.autotune(...) <- foo.__code__.co_firstlineno + # @triton.heuristics(...) + # @triton.jit + # def foo(...): <- this line is the first line + for idx, line in enumerate(lines): + if line.strip().startswith("def "): + begin_line += idx + break + return file_name, begin_line diff --git a/lib/python3.12/site-packages/triton/testing.py b/lib/python3.12/site-packages/triton/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..71cb8ab1eaaace9f9cae6a6e888d9c47a20115fd --- /dev/null +++ b/lib/python3.12/site-packages/triton/testing.py @@ -0,0 +1,511 @@ +import functools +import os +import subprocess +import sys +from contextlib import contextmanager +from typing import Any, Dict, List +from . import language as tl +from . import runtime + + +def nvsmi(attrs): + attrs = ','.join(attrs) + cmd = ['nvidia-smi', '-i', '0', '--query-gpu=' + attrs, '--format=csv,noheader,nounits'] + out = subprocess.check_output(cmd) + ret = out.decode(sys.stdout.encoding).split(',') + ret = [int(x) for x in ret] + return ret + + +def _summarize_statistics(times, quantiles, return_mode): + import torch + if quantiles is not None: + ret = torch.quantile(times, torch.tensor(quantiles, dtype=torch.float)).tolist() + if len(ret) == 1: + ret = ret[0] + return ret + if return_mode == "all": + return times.tolist() + return getattr(torch, return_mode)(times).item() + + +def do_bench_cudagraph(fn, rep=20, grad_to_none=None, quantiles=None, return_mode="mean"): + """ + Benchmark the runtime of the provided function. + + :param fn: Function to benchmark + :type fn: Callable + :param rep: Repetition time (in ms) + :type rep: int + :param grad_to_none: Reset the gradient of the provided tensor to None + :type grad_to_none: torch.tensor, optional + :param return_mode: The statistical measure to return. Options are "min", "max", "mean", "median", or "all" Default is "mean". + :type return_mode: str + """ + import torch + assert return_mode in ["min", "max", "mean", "median", "all"] + + with torch.cuda.stream(torch.cuda.Stream()): + # warmup + fn() + if grad_to_none is not None: + for x in grad_to_none: + x.detach_() + x.requires_grad_(True) + x.grad = None + # step 1 - we estimate the amount of time the kernel call takes + # NOTE: this estimate isn't super accurate because the GPU isn't warmed up at this point + # but it is probably good enough + # NOTE: we don't use a graph to estimate the runtime because creating a graph is expensive, + # ~300ms on A100, so we default to the same method used in `do_bench` (minus the L2 + # cache flush). + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + for _ in range(5): + fn() + end_event.record() + torch.cuda.synchronize() + estimate_ms = start_event.elapsed_time(end_event) / 5 + n_repeat = max(1, int(rep / estimate_ms)) + # step 2 - construct a cuda graph with `n_repeat` unrolled function calls to minimize + # host overhead + g = torch.cuda.CUDAGraph() + with torch.cuda.graph(g): + for _ in range(n_repeat): + if grad_to_none is not None: + for x in grad_to_none: + x.grad = None + fn() + torch.cuda.synchronize() + # measure time and return + ret = [] + n_retries = 10 + for _ in range(n_retries): + start_event = torch.cuda.Event(enable_timing=True) + end_event = torch.cuda.Event(enable_timing=True) + start_event.record() + g.replay() + end_event.record() + torch.cuda.synchronize() + ret += [start_event.elapsed_time(end_event) / n_repeat] + return _summarize_statistics(torch.tensor(ret), quantiles, return_mode) + + +def do_bench(fn, warmup=25, rep=100, grad_to_none=None, quantiles=None, return_mode="mean"): + """ + Benchmark the runtime of the provided function. By default, return the median runtime of :code:`fn` along with + the 20-th and 80-th performance percentile. + + :param fn: Function to benchmark + :type fn: Callable + :param warmup: Warmup time (in ms) + :type warmup: int + :param rep: Repetition time (in ms) + :type rep: int + :param grad_to_none: Reset the gradient of the provided tensor to None + :type grad_to_none: torch.tensor, optional + :param quantiles: Performance percentile to return in addition to the median. + :type quantiles: list[float], optional + :param return_mode: The statistical measure to return. Options are "min", "max", "mean", "median", or "all" Default is "mean". :type return_mode: str + """ + assert return_mode in ["min", "max", "mean", "median", "all"] + import torch + + di = runtime.driver.active.get_device_interface() + + fn() + di.synchronize() + + cache = runtime.driver.active.get_empty_cache_for_benchmark() + + # Estimate the runtime of the function + start_event = di.Event(enable_timing=True) + end_event = di.Event(enable_timing=True) + start_event.record() + for _ in range(5): + cache.zero_() + fn() + end_event.record() + di.synchronize() + estimate_ms = start_event.elapsed_time(end_event) / 5 + + # compute number of warmup and repeat + n_warmup = max(1, int(warmup / estimate_ms)) + n_repeat = max(1, int(rep / estimate_ms)) + start_event = [di.Event(enable_timing=True) for i in range(n_repeat)] + end_event = [di.Event(enable_timing=True) for i in range(n_repeat)] + # Warm-up + for _ in range(n_warmup): + fn() + # Benchmark + for i in range(n_repeat): + # we don't want `fn` to accumulate gradient values + # if it contains a backward pass. So we clear the + # provided gradients + if grad_to_none is not None: + for x in grad_to_none: + x.grad = None + # we clear the L2 cache before each run + cache.zero_() + # record time of `fn` + start_event[i].record() + fn() + end_event[i].record() + # Record clocks + di.synchronize() + times = torch.tensor([s.elapsed_time(e) for s, e in zip(start_event, end_event)], dtype=torch.float) + return _summarize_statistics(times, quantiles, return_mode) + + +def assert_close(x, y, atol=None, rtol=None, err_msg=''): + """ + Asserts that two inputs are close within a certain tolerance. + + :param x: The first input. + :type x: scala, list, numpy.ndarray, or torch.Tensor + :param y: The second input. + :type y: scala, list, numpy.ndarray, or torch.Tensor + :param atol: The absolute tolerance. Default value is 1e-2. + :type atol: float, optional + :param rtol: The relative tolerance. Default value is 0. + :type rtol: float, optional + :param err_msg: The error message to use if the assertion fails. + :type err_msg: str + """ + import numpy as np + import torch + + # canonicalize arguments to be tensors + if not isinstance(x, torch.Tensor): + x = torch.tensor(x) + if not isinstance(y, torch.Tensor): + y = torch.tensor(y) + # absolute tolerance + if atol is None: + atol = 1e-2 + atol = atol(x.dtype) if callable(atol) else atol + # relative tolerance hook + if rtol is None: + rtol = 0. + rtol = rtol(x.dtype) if callable(rtol) else rtol + # we use numpy instead of pytorch + # as it seems more memory efficient + # pytorch tends to oom on large tensors + if isinstance(x, torch.Tensor): + if x.dtype == torch.bfloat16: + x = x.float() + x = x.cpu().detach().numpy() + if isinstance(y, torch.Tensor): + if y.dtype == torch.bfloat16: + y = y.float() + y = y.cpu().detach().numpy() + # we handle size==1 case separately as we can + # provide better error message there + if x.size > 1 or y.size > 1: + np.testing.assert_allclose(x, y, atol=atol, rtol=rtol, equal_nan=True) + return + if not np.allclose(x, y, atol=atol, rtol=rtol): + raise AssertionError(f'{err_msg} {x} is not close to {y} (atol={atol}, rtol={rtol})') + + +class Benchmark: + """ + This class is used by the :code:`perf_report` function to generate line plots with a concise API. + """ + + def __init__( + self, + x_names: List[str], + x_vals: List[Any], + line_arg: str, + line_vals: List[Any], + line_names: List[str], + plot_name: str, + args: Dict[str, Any], + xlabel: str = '', + ylabel: str = '', + x_log: bool = False, + y_log: bool = False, + styles=None, + ): + """ + Constructor. + x_vals can be a list of scalars or a list of tuples/lists. If x_vals is a list + of scalars and there are multiple x_names, all arguments will have the same value. + If x_vals is a list of tuples/lists, each element should have the same length as + x_names. + + :param x_names: Name of the arguments that should appear on the x axis of the plot. + :type x_names: List[str] + :param x_vals: List of values to use for the arguments in :code:`x_names`. + :type x_vals: List[Any] + :param line_arg: Argument name for which different values correspond to different lines in the plot. + :type line_arg: str + :param line_vals: List of values to use for the arguments in :code:`line_arg`. + :type line_vals: List[Any] + :param line_names: Label names for the different lines. + :type line_names: List[str] + :param plot_name: Name of the plot. + :type plot_name: str + :param args: Dictionary of keyword arguments to remain fixed throughout the benchmark. + :type args: Dict[str, Any] + :param xlabel: Label for the x axis of the plot. + :type xlabel: str, optional + :param ylabel: Label for the y axis of the plot. + :type ylabel: str, optional + :param x_log: Whether the x axis should be log scale. + :type x_log: bool, optional + :param y_log: Whether the y axis should be log scale. + :type y_log: bool, optional + :param styles: A list of tuples, where each tuple contains two elements: a color and a linestyle. + :type styles: list[tuple[str, str]] + """ + self.x_names = x_names + self.x_vals = x_vals + self.x_log = x_log + self.line_arg = line_arg + self.line_vals = line_vals + self.line_names = line_names + self.y_log = y_log + self.styles = styles + # plot info + self.xlabel = xlabel + self.ylabel = ylabel + self.plot_name = plot_name + self.args = args + + +class Mark: + + def __init__(self, fn, benchmarks): + self.fn = fn + self.benchmarks = benchmarks + + def _run(self, bench: Benchmark, save_path: str, show_plots: bool, print_data: bool, diff_col=False, + save_precision=6, **kwrags): + import os + + import matplotlib.pyplot as plt + import pandas as pd + y_mean = bench.line_names + y_min = [f'{x}-min' for x in bench.line_names] + y_max = [f'{x}-max' for x in bench.line_names] + x_names = list(bench.x_names) + df = pd.DataFrame(columns=x_names + y_mean + y_min + y_max) + for x in bench.x_vals: + # x can be a single value or a sequence of values. + if not isinstance(x, (list, tuple)): + x = [x for _ in x_names] + + if len(x) != len(x_names): + raise ValueError(f"Expected {len(x_names)} values, got {x}") + x_args = dict(zip(x_names, x)) + + row_mean, row_min, row_max = [], [], [] + for y in bench.line_vals: + ret = self.fn(**x_args, **{bench.line_arg: y}, **bench.args, **kwrags) + try: + y_mean, y_min, y_max = ret + except TypeError: + y_mean, y_min, y_max = ret, None, None + row_mean += [y_mean] + row_min += [y_min] + row_max += [y_max] + df.loc[len(df)] = list(x) + row_mean + row_min + row_max + + if bench.plot_name: + plt.figure() + ax = plt.subplot() + # Plot first x value on x axis if there are multiple. + first_x = x_names[0] + for i, y in enumerate(bench.line_names): + y_min, y_max = df[y + '-min'], df[y + '-max'] + col = bench.styles[i][0] if bench.styles else None + sty = bench.styles[i][1] if bench.styles else None + ax.plot(df[first_x], df[y], label=y, color=col, ls=sty) + if not y_min.isnull().all() and not y_max.isnull().all(): + y_min = y_min.astype(float) + y_max = y_max.astype(float) + ax.fill_between(df[first_x], y_min, y_max, alpha=0.15, color=col) + ax.legend() + ax.set_xlabel(bench.xlabel or first_x) + ax.set_ylabel(bench.ylabel) + # ax.set_title(bench.plot_name) + ax.set_xscale("log" if bench.x_log else "linear") + ax.set_yscale("log" if bench.y_log else "linear") + if show_plots: + plt.show() + if save_path: + plt.savefig(os.path.join(save_path, f"{bench.plot_name}.png")) + df = df[x_names + bench.line_names] + if diff_col and df.shape[1] == 2: + col0, col1 = df.columns.tolist() + df['Diff'] = df[col1] - df[col0] + + if print_data: + print(bench.plot_name + ':') + print(df.to_string()) + if save_path: + df.to_csv(os.path.join(save_path, f"{bench.plot_name}.csv"), float_format=f"%.{save_precision}f", + index=False) + return df + + def run(self, show_plots=False, print_data=False, save_path='', return_df=False, **kwargs): + has_single_bench = isinstance(self.benchmarks, Benchmark) + benchmarks = [self.benchmarks] if has_single_bench else self.benchmarks + result_dfs = [] + if save_path: + # Create directory if it doesn't exist + os.makedirs(save_path, exist_ok=True) + html = open(os.path.join(save_path, "results.html"), "w") + html.write("\n") + for bench in benchmarks: + result_dfs.append(self._run(bench, save_path, show_plots, print_data, **kwargs)) + if save_path: + html.write(f"\n") + if save_path: + html.write("\n") + html.close() + if return_df: + if has_single_bench: + return result_dfs[0] + else: + return result_dfs + return None + + +def perf_report(benchmarks): + """ + Mark a function for benchmarking. The benchmark can then be executed by using the :code:`.run` method on the return value. + + :param benchmarks: Benchmarking configurations. + :type benchmarks: List of :class:`Benchmark` + """ + wrapper = lambda fn: Mark(fn, benchmarks) + return wrapper + + +def get_dram_gbps(device=None): + ''' return DRAM bandwidth in GB/s ''' + import torch + + from .runtime import driver + if not device: + device = torch.cuda.current_device() + mem_clock_khz = driver.active.utils.get_device_properties(device)["mem_clock_rate"] # in kHz + bus_width = driver.active.utils.get_device_properties(device)["mem_bus_width"] + bw_gbps = mem_clock_khz * bus_width * 2 / 1e6 / 8 # In GB/s + return bw_gbps + + +def get_max_tensorcore_tflops(dtype, clock_rate, device=None): + import torch + + from .runtime import driver + if not device: + device = torch.cuda.current_device() + + num_subcores = driver.active.utils.get_device_properties(device)["multiprocessor_count"] * 4 + capability = torch.cuda.get_device_capability(device) + if capability[0] < 8: + assert dtype == torch.float16 + ops_per_sub_core = 256 # 2 4x4x4 Tensor Cores + else: + if dtype in [torch.float32, torch.int32]: + ops_per_sub_core = 256 + elif dtype in [torch.float16, torch.bfloat16, torch.int16]: + ops_per_sub_core = 512 + elif dtype in [torch.int8, tl.float8e4nv, tl.float8e4b15, tl.float8e5]: + ops_per_sub_core = 1024 + else: + raise RuntimeError("dtype not supported") + tflops = num_subcores * clock_rate * ops_per_sub_core * 1e-9 + return tflops + + +# create decorator that wraps test function into +# a cuda-memcheck system call + + +def cuda_memcheck(**target_kwargs): + + def decorator(test_fn): + + @functools.wraps(test_fn) + def wrapper(*args, **kwargs): + import psutil + ppid_name = psutil.Process(os.getppid()).name() + run_cuda_memcheck = target_kwargs.items() <= kwargs.items() + if run_cuda_memcheck and ppid_name != "cuda-memcheck": + path = os.path.realpath(test_fn.__globals__["__file__"]) + # get path of current file + env = {"PATH": os.environ["PATH"], "PYTORCH_NO_CUDA_MEMORY_CACHING": "1"} + assert 'request' in kwargs, "memcheck'ed test must have a (possibly unused) `request` fixture" + test_id = kwargs['request'].node.callspec.id + cmd = f"{path}::{test_fn.__name__}[{test_id}]" + out = subprocess.run(["cuda-memcheck", "pytest", "-vs", cmd], capture_output=True, env=env) + assert out.returncode == 0, "cuda-memcheck returned an error: bounds checking failed" + assert "ERROR SUMMARY: 0 errors" in str(out.stdout) + else: + test_fn(*args, **kwargs) + + return wrapper + + return decorator + + +@contextmanager +def set_gpu_clock(ref_sm_clock=1350, ref_mem_clock=1215): + try: + subprocess.check_output(["nvidia-smi", "-i", "0", "-pm", "1"]) + subprocess.check_output([ + "nvidia-smi", + "-i", + "0", + f"--lock-gpu-clocks={ref_sm_clock},{ref_sm_clock}", + ]) + subprocess.check_output([ + "nvidia-smi", + "-i", + "0", + f"--lock-memory-clocks={ref_mem_clock},{ref_mem_clock}", + ]) + cur_sm_clock = nvsmi(["clocks.current.sm"])[0] + cur_mem_clock = nvsmi(["clocks.current.memory"])[0] + assert abs(cur_sm_clock - ref_sm_clock) < 10, f"GPU SMs must run at {ref_sm_clock} MHz" + assert abs(cur_mem_clock - ref_mem_clock) < 10, f"GPU SMs must run at {ref_mem_clock} MHz" + tflops = 1e-6 * 2 * 108 * 4 * 256 * ref_sm_clock + gbps = 640 * 2 * ref_mem_clock * 1e-3 + yield tflops, gbps + finally: + subprocess.check_output(["nvidia-smi", "-i", "0", "-pm", "0"]) + subprocess.check_output(["nvidia-smi", "-i", "0", "-rgc"]) + subprocess.check_output(["nvidia-smi", "-i", "0", "-rmc"]) + + +def get_max_simd_tflops(dtype, clock_rate, device=None): + import torch + + from .runtime import driver + if not device: + device = torch.cuda.current_device() + + num_subcores = driver.active.utils.get_device_properties(device)["multiprocessor_count"] * 4 + capability = torch.cuda.get_device_capability() + if capability[0] < 8: + if dtype == torch.float32: + ops_per_sub_core = 32 # 2*16 + elif dtype == torch.float16: + ops_per_sub_core = 64 + else: + raise RuntimeError("dtype not supported") + else: + if dtype == torch.float32: + ops_per_sub_core = 32 + elif dtype in [torch.float16, torch.bfloat16]: + ops_per_sub_core = 64 + else: + raise RuntimeError("dtype not supported") + tflops = num_subcores * clock_rate * ops_per_sub_core * 1e-9 + return tflops diff --git a/lib/python3.12/site-packages/triton/tools/__init__.py b/lib/python3.12/site-packages/triton/tools/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/triton/tools/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/triton/tools/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..20cc676d28e76e6807e224686da87259bfaf6987 Binary files /dev/null and b/lib/python3.12/site-packages/triton/tools/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/tools/__pycache__/build_extern.cpython-312.pyc b/lib/python3.12/site-packages/triton/tools/__pycache__/build_extern.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..a7761338e3899f20101eec3b945754f7a185fdb3 Binary files /dev/null and b/lib/python3.12/site-packages/triton/tools/__pycache__/build_extern.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/tools/__pycache__/compile.cpython-312.pyc b/lib/python3.12/site-packages/triton/tools/__pycache__/compile.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..da755d8bc69c70b2753136c4f4593366305f1eed Binary files /dev/null and b/lib/python3.12/site-packages/triton/tools/__pycache__/compile.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/tools/__pycache__/disasm.cpython-312.pyc b/lib/python3.12/site-packages/triton/tools/__pycache__/disasm.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..fadaf9157ec86c6a41d7249a18e10f98370399a1 Binary files /dev/null and b/lib/python3.12/site-packages/triton/tools/__pycache__/disasm.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/tools/__pycache__/experimental_descriptor.cpython-312.pyc b/lib/python3.12/site-packages/triton/tools/__pycache__/experimental_descriptor.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..5cfd8d26edd71c5eca45566543d8a17e4bb409a8 Binary files /dev/null and b/lib/python3.12/site-packages/triton/tools/__pycache__/experimental_descriptor.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/tools/__pycache__/link.cpython-312.pyc b/lib/python3.12/site-packages/triton/tools/__pycache__/link.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8262bc5b64fcf74bdae683e38c672e39605557db Binary files /dev/null and b/lib/python3.12/site-packages/triton/tools/__pycache__/link.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/triton/tools/build_extern.py b/lib/python3.12/site-packages/triton/tools/build_extern.py new file mode 100644 index 0000000000000000000000000000000000000000..8f0168d59d7af045bde68a508a000654f4893bb1 --- /dev/null +++ b/lib/python3.12/site-packages/triton/tools/build_extern.py @@ -0,0 +1,365 @@ +import argparse +import subprocess +from abc import ABC, abstractmethod +from typing import Dict, List, Optional + + +class Symbol: + _name: str + _op_name: str + _ret_type: str + _arg_names: List[str] + _arg_types: List[str] + + def __init__( + self, + name: str, + op_name: str, + ret_type: str, + arg_names: List[str], + arg_types: List[str], + ) -> None: + ''' + A symbol is a function declaration. + :param name: name of the symbol + :param op_name: name of the operation + :param ret_type: return type of the operation + :param arg_names: names of the arguments + :param arg_types: types of the arguments + ''' + self._name = name + self._op_name = op_name + self._ret_type = ret_type + self._arg_names = list(arg_names) + self._arg_types = list(arg_types) + + @property + def name(self) -> str: + return self._name + + @property + def op_name(self) -> str: + return self._op_name + + @property + def ret_type(self) -> str: + return self._ret_type + + @property + def arg_names(self) -> List[str]: + return self._arg_names + + @property + def arg_types(self) -> List[str]: + return self._arg_types + + +def convert_type(type_str) -> Optional[str]: + if type_str == "i32": + return "int32" + elif type_str == "u32": + return "uint32" + elif type_str == "i64": + return "int64" + elif type_str == "u64": + return "uint64" + elif type_str == "float": + return "fp32" + elif type_str == "double": + return "fp64" + else: + # ignore other types, such as pointer types + return None + + +def to_unsigned(type_str) -> str: + if type_str == "int32": + return "uint32" + elif type_str == "int64": + return "uint64" + else: + return type_str + + +class ExternLibrary(ABC): + _name: str + _path: str + _symbols: Dict[str, Symbol] + _format: bool + _grouping: bool + + def __init__( + self, + name: str, + path: str, + format: bool = True, + grouping: bool = True, + ) -> None: + ''' + Abstract class for extern library. + :param name: name of the library + :param path: path of the library + :param format: whether to format the generated stub file + ''' + self._name = name + self._path = path + self._symbols = {} + self._format = format + self._grouping = grouping + + @property + def name(self) -> str: + return self._name + + @property + def path(self) -> str: + return self._path + + @property + def symbols(self) -> Dict[str, Symbol]: + return self._symbols + + @property + def grouping(self) -> bool: + return self._grouping + + @abstractmethod + def parse_symbols(self, input_file) -> None: + pass + + @abstractmethod + def _output_stubs(self) -> str: + pass + + def generate_stub_file(self, output_dir) -> None: + file_str = self._output_stubs() + if file_str is None or len(file_str) == 0: + raise Exception("file_str is empty") + + output_file = f"{output_dir}/{self._name}.py" + with open(output_file, "w") as f: + f.write(file_str) + f.close() + if self._format: + subprocess.Popen(["autopep8", "-a", "-r", "-i", output_file], stdout=subprocess.PIPE).communicate() + subprocess.Popen(["isort", output_file], stdout=subprocess.PIPE).communicate() + + +class Libdevice(ExternLibrary): + _symbol_groups: Dict[str, List[Symbol]] + + def __init__(self, path) -> None: + ''' + Constructor for Libdevice. + :param path: path of the libdevice library + ''' + super().__init__("libdevice", path) + self._symbol_groups = {} + self.is_pure = True + + @staticmethod + def _extract_symbol(line) -> Optional[Symbol]: + # Extract symbols from line in the following format: + # "define [internal] @(,)" + entries = line.split("@") + ret_str = entries[0] + func_str = entries[1] + # Get ret_type, skip internal symbols + ret_strs = ret_str.split() + if ret_strs[1] == "internal": + return None + ret_type = convert_type(ret_strs[1]) + if ret_type is None: + return None + # Get function name + func_strs = func_str.split("(") + func_name = func_strs[0].replace("@", "") + op_name = func_name.replace("__nv_", "") + if 'ieee' in op_name: + return None + # Get arg_types + arg_strs = func_strs[1].split(",") + arg_types = [] + arg_names = [] + for i, arg_str in enumerate(arg_strs): + arg_type = convert_type(arg_str.split()[0]) + if arg_type is None: + return None + arg_name = 'arg' + str(i) + arg_types.append(arg_type) + arg_names.append(arg_name) + if op_name == "sad": + # Special case for sad, where the last argument is an unsigned int + arg_types[-1] = to_unsigned(arg_types[-1]) + elif op_name.startswith("u"): + # LLVM does not differentiate between signed and unsigned integer type. + # We have to convert the types to unsigned + ret_type = to_unsigned(ret_type) + for i, arg_type in enumerate(arg_types): + arg_types[i] = to_unsigned(arg_type) + return Symbol(func_name, op_name, ret_type, arg_names, arg_types) + + def _group_symbols(self) -> None: + symbol_set = {} + for symbol in self._symbols.values(): + op_name = symbol.op_name + symbol_set[op_name] = symbol + + # Group functions together by renaming. + renaming = { + 'llabs': 'abs', 'acosf': 'acos', 'acoshf': 'acosh', 'dadd_rd': 'add_rd', 'fadd_rd': 'add_rd', 'dadd_rn': + 'add_rn', 'fadd_rn': 'add_rn', 'dadd_ru': 'add_ru', 'fadd_ru': 'add_ru', 'dadd_rz': 'add_rz', 'fadd_rz': + 'add_rz', 'asinf': 'asin', 'asinhf': 'asinh', 'atanf': 'atan', 'atan2f': 'atan2', 'atanhf': 'atanh', + 'brevll': 'brev', 'cbrtf': 'cbrt', 'ceilf': 'ceil', 'clzll': 'clz', 'copysignf': 'copysign', 'cosf': 'cos', + 'coshf': 'cosh', 'cospif': 'cospi', 'cyl_bessel_i0f': 'cyl_bessel_i0', 'cyl_bessel_i1f': 'cyl_bessel_i1', + 'fdiv_rd': 'div_rd', 'ddiv_rd': 'div_rd', 'fdiv_rn': 'div_rn', 'ddiv_rn': 'div_rn', 'fdiv_ru': 'div_ru', + 'ddiv_ru': 'div_ru', 'fdiv_rz': 'div_rz', 'ddiv_rz': 'div_rz', 'erff': 'erf', 'erfcf': 'erfc', 'erfcinvf': + 'erfcinv', 'erfcxf': 'erfcx', 'erfinvf': 'erfinv', 'expf': 'exp', 'exp10f': 'exp10', 'exp2f': 'exp2', + 'expm1f': 'expm1', 'fabsf': 'abs', 'fabs': 'abs', 'fast_fdividef': 'fast_dividef', 'fdimf': 'fdim', 'ffsll': + 'ffs', 'floorf': 'floor', 'fmaf': 'fma', 'fmaf_rd': 'fma_rd', 'fmaf_rn': 'fma_rn', 'fmaf_ru': 'fma_ru', + 'fmaf_rz': 'fma_rz', 'fmodf': 'fmod', 'uhadd': 'hadd', 'hypotf': 'hypot', 'ilogbf': 'ilogb', 'isinff': + 'isinf', 'isinfd': 'isinf', 'isnanf': 'isnan', 'isnand': 'isnan', 'j0f': 'j0', 'j1f': 'j1', 'jnf': 'jn', + 'ldexpf': 'ldexp', 'lgammaf': 'lgamma', 'llrintf': 'llrint', 'llroundf': 'llround', 'logf': 'log', 'log10f': + 'log10', 'log1pf': 'log1p', 'log2f': 'log2', 'logbf': 'logb', 'umax': 'max', 'llmax': 'max', 'ullmax': + 'max', 'fmaxf': 'max', 'fmax': 'max', 'umin': 'min', 'llmin': 'min', 'ullmin': 'min', 'fminf': 'min', + 'fmin': 'min', 'dmul_rd': 'mul_rd', 'fmul_rd': 'mul_rd', 'dmul_rn': 'mul_rn', 'fmul_rn': 'mul_rn', + 'dmul_ru': 'mul_ru', 'fmul_ru': 'mul_ru', 'dmul_rz': 'mul_rz', 'fmul_rz': 'mul_rz', 'umul24': 'mul24', + 'umulhi': 'mulhi', 'mul64hi': 'mulhi', 'umul64hi': 'mulhi', 'nearbyintf': 'nearbyint', 'nextafterf': + 'nextafter', 'norm3df': 'norm3d', 'norm4df': 'norm4d', 'normcdff': 'normcdf', 'normcdfinvf': 'normcdfinv', + 'popcll': 'popc', 'powif': 'pow', 'powi': 'pow', 'powf': 'pow', 'rcbrtf': 'rcbrt', 'frcp_rd': 'rcp_rd', + 'drcp_rd': 'rcp_rd', 'frcp_rn': 'rcp_rn', 'drcp_rn': 'rcp_rn', 'frcp_ru': 'rcp_ru', 'drcp_ru': 'rcp_ru', + 'frcp_rz': 'rcp_rz', 'drcp_rz': 'rcp_rz', 'remainderf': 'remainder', 'urhadd': 'rhadd', 'rhypotf': 'rhypot', + 'rintf': 'rint', 'rnorm3df': 'rnorm3d', 'rnorm4df': 'rnorm4d', 'roundf': 'round', 'rsqrtf': 'rsqrt', + 'frsqrt_rn': 'rsqrt_rn', 'usad': 'sad', 'scalbnf': 'scalbn', 'signbitf': 'signbit', 'signbitd': 'signbit', + 'sinf': 'sin', 'sinhf': 'sinh', 'sinpif': 'sinpi', 'sqrtf': 'sqrt', 'fsqrt_rd': 'sqrt_rd', 'dsqrt_rd': + 'sqrt_rd', 'fsqrt_rn': 'sqrt_rn', 'dsqrt_rn': 'sqrt_rn', 'fsqrt_ru': 'sqrt_ru', 'dsqrt_ru': 'sqrt_ru', + 'fsqrt_rz': 'sqrt_rz', 'dsqrt_rz': 'sqrt_rz', 'fsub_rd': 'sub_rd', 'dsub_rd': 'sub_rd', 'fsub_rn': 'sub_rn', + 'dsub_rn': 'sub_rn', 'fsub_ru': 'sub_ru', 'dsub_ru': 'sub_ru', 'fsub_rz': 'sub_rz', 'dsub_rz': 'sub_rz', + 'tanf': 'tan', 'tanhf': 'tanh', 'tgammaf': 'tgamma', 'truncf': 'trunc', 'y0f': 'y0', 'y1f': 'y1', 'ynf': + 'yn' + } + + for symbol in self._symbols.values(): + op_name = symbol.op_name + if op_name in renaming: + op_name = renaming[op_name] + symbol._op_name = op_name + if op_name in self._symbol_groups: + self._symbol_groups[op_name].append(symbol) + else: + self._symbol_groups[op_name] = [symbol] + + def parse_symbols(self, input_file) -> None: + if len(self.symbols) > 0: + return + output = subprocess.check_output(["grep", "define", input_file]).decode().splitlines() + for line in output: + symbol = self._extract_symbol(line) + if symbol is None: + continue + self._symbols[symbol.name] = symbol + + self._group_symbols() + + def _output_stubs(self) -> str: + # Generate python functions in the following format: + # @extern.extern + # def (, _builder=None): + # arg_type_symbol_dict = {[arg_type]: {(symbol, ret_type)}} + # return core.extern_elementwise("libdevice", , , , _builder) + import_str = "from . import core\n" + + header_str = "" + func_str = "" + for symbols in self._symbol_groups.values(): + func_str += "@core.extern\n" + func_name_str = f"def {symbols[0].op_name}(" + for arg_name in symbols[0].arg_names: + func_name_str += f"{arg_name}, " + func_name_str += "_builder=None):\n" + + return_str = f"\treturn core.extern_elementwise(\"{self._name}\", libdevice_path(), [" + for arg_name in symbols[0].arg_names: + return_str += f"{arg_name}, " + return_str += "], \n" + + arg_type_symbol_dict_str = "{" + for symbol in symbols: + arg_type_symbol_dict_str += "(" + for arg_type in symbol.arg_types: + arg_type_symbol_dict_str += f'core.dtype("{arg_type}"),' + ret_type = f'core.dtype("{symbol.ret_type}")' + arg_type_symbol_dict_str += "): (\"" + symbol.name + "\", " + ret_type + "),\n" + arg_type_symbol_dict_str += "}" + + return_str += arg_type_symbol_dict_str + return_str += f", is_pure={self.is_pure}" + return_str += ", _builder=_builder)\n" + + func_str += func_name_str + return_str + "\n" + file_str = import_str + header_str + func_str + + return file_str + + +class LLVMDisassembler: + _path: str + _ll_file: str + + def __init__(self, path) -> None: + ''' + Invoke llvm-dis to disassemble the given file. + :param path: path to llvm-dis + ''' + self._path = path + self._ll_file = "/tmp/extern_lib.ll" + + def disasm(self, lib_path: str) -> None: + subprocess.Popen([self._path, lib_path, "-o", self.ll_file], stdout=subprocess.PIPE).communicate() + + @property + def ll_file(self) -> str: + return self._ll_file + + @property + def path(self) -> str: + return self._path + + +extern_libs = ["libdevice"] + + +def build( + llvm_dis_path: str, + lib_path: str, + lib_name: str, + output_dir: str, +) -> None: + ''' + Interface function to build the library file. + :param llvm_dis_path: path to the llvm-dis binary + :param lib_path: path to the external library file + :param lib_name: name of the library + :param output_dir: path to the output directory + ''' + if lib_name == "libdevice": + extern_lib = Libdevice(lib_path) + else: + raise Exception(f"Unknown extern library: {lib_name}") + + llvm_disassembler = LLVMDisassembler(llvm_dis_path) + llvm_disassembler.disasm(lib_path) + + extern_lib.parse_symbols(llvm_disassembler.ll_file) + extern_lib.generate_stub_file(output_dir) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--llvm-dis", dest="llvm_dis_path", help="Path to llvm-dis", default="llvm-dis") + parser.add_argument("--lib-path", dest="lib_path", help="Path to the extern library") + parser.add_argument("--lib-name", dest="lib_name", help="Name of the extern library") + parser.add_argument("--output", dest="output_dir", help="Output file path", default="/tmp/") + args = parser.parse_args() + + build(args.llvm_dis_path, args.lib_path, args.lib_name, args.output_dir) diff --git a/lib/python3.12/site-packages/triton/tools/compile.c b/lib/python3.12/site-packages/triton/tools/compile.c new file mode 100644 index 0000000000000000000000000000000000000000..971bf61912a752465c83a0b17ae131bee6f5cc74 --- /dev/null +++ b/lib/python3.12/site-packages/triton/tools/compile.c @@ -0,0 +1,67 @@ +/* clang-format off */ +#include +#include +#include +#include +#include + + +// helpers to check for cuda errors +#define CUDA_CHECK(ans) {{\ + gpuAssert((ans), __FILE__, __LINE__);\ + }}\ + +static inline void gpuAssert(CUresult code, const char *file, int line) {{ + if (code != CUDA_SUCCESS) {{ + const char *prefix = "Triton Error [CUDA]: "; + const char *str; + cuGetErrorString(code, &str); + char err[1024] = {{0}}; + strcat(err, prefix); + strcat(err, str); + printf("%s\\n", err); + exit(code); + }} +}} + +// globals +#define CUBIN_NAME {kernel_name}_cubin +CUmodule {kernel_name}_mod = NULL; +CUfunction {kernel_name}_func = NULL; +unsigned char CUBIN_NAME[{bin_size}] = {{ {bin_data} }}; + + +void unload_{kernel_name}(void) {{ + CUDA_CHECK(cuModuleUnload({kernel_name}_mod)); +}} + +// TODO: some code duplication with `runtime/backend/cuda.c` +void load_{kernel_name}() {{ + int dev = 0; + void *bin = (void *)&CUBIN_NAME; + int shared = {shared}; + CUDA_CHECK(cuModuleLoadData(&{kernel_name}_mod, bin)); + CUDA_CHECK(cuModuleGetFunction(&{kernel_name}_func, {kernel_name}_mod, "{triton_kernel_name}")); + // set dynamic shared memory if necessary + int shared_optin; + CUDA_CHECK(cuDeviceGetAttribute(&shared_optin, CU_DEVICE_ATTRIBUTE_MAX_SHARED_MEMORY_PER_BLOCK_OPTIN, dev)); + if (shared > 49152 && shared_optin > 49152) {{ + CUDA_CHECK(cuFuncSetCacheConfig({kernel_name}_func, CU_FUNC_CACHE_PREFER_SHARED)); + CUDA_CHECK(cuFuncSetAttribute({kernel_name}_func, CU_FUNC_ATTRIBUTE_MAX_DYNAMIC_SHARED_SIZE_BYTES, shared_optin)) + }} +}} + +/* +{kernel_docstring} +*/ +CUresult {kernel_name}(CUstream stream, {signature}) {{ + if ({kernel_name}_func == NULL) + load_{kernel_name}(); + unsigned int gX = {gridX}; + unsigned int gY = {gridY}; + unsigned int gZ = {gridZ}; + void *args[{num_args}] = {{ {arg_pointers} }}; + // TODO: shared memory + if(gX * gY * gZ > 0) + return cuLaunchKernel({kernel_name}_func, gX, gY, gZ, {num_warps} * 32, 1, 1, {shared}, stream, args, NULL); +}} diff --git a/lib/python3.12/site-packages/triton/tools/compile.h b/lib/python3.12/site-packages/triton/tools/compile.h new file mode 100644 index 0000000000000000000000000000000000000000..d98b7063b6ae6292b65b61abf5a30c58b7d28e95 --- /dev/null +++ b/lib/python3.12/site-packages/triton/tools/compile.h @@ -0,0 +1,14 @@ +#ifndef TT_KERNEL_INCLUDES +#define TT_KERNEL_INCLUDES + +#include +#include +#include +#include + +#endif + +void unload_{kernel_name}(void); +void load_{kernel_name}(void); +// tt-linker: {kernel_name}:{full_signature}:{algo_info} +CUresult{_placeholder} {kernel_name}(CUstream stream, {signature}); diff --git a/lib/python3.12/site-packages/triton/tools/compile.py b/lib/python3.12/site-packages/triton/tools/compile.py new file mode 100644 index 0000000000000000000000000000000000000000..443341fa0d474610d758855053cf5fc25a6ea68d --- /dev/null +++ b/lib/python3.12/site-packages/triton/tools/compile.py @@ -0,0 +1,155 @@ +import binascii +import hashlib +import importlib.util +import sys +from argparse import ArgumentParser +from pathlib import Path +from typing import List + +import triton +import triton.backends +from triton.compiler.code_generator import kernel_suffix +from triton.backends.nvidia.driver import ty_to_cpp + +desc = """ +Triton ahead-of-time compiler: + +This program compiles the kernel with name `kernel-name` in the file at the +provided `path` into self-contained C source-code that embeds the `cubin` +data along with utilities to load, unload and launch the kernel. + +signature is provided as a list of (optionally divisibility-hinted) types +or constexpr values, e.g. + +`compile.py --kernel-name kernel --signature "*fp32:16, i32:16, 1024, i32" --out-name kernel /path/to/kernel.py` + +will compile triton.JITFunction of name `kernel` inside the file `/path/to/kernel.py`. +Said kernel will be specialized such that argument 0, 1 are assumed to be multiple of 16, +and argument 2 is assumed to be a compile-time constant of value 1024, i.e. it won't be part of the generated prototype. + +The resulting entry point will have signature + +CUresult kernel_{specialization_suffix}(CUstream stream, unsigned gX, unsigned gY, unsigned gZ, float* arg0, int32_t arg1, int32_t arg2) + +Different such specialized entry points can be combined using the `linker.py` script. + +NOTE: when resolving the scope of /path/to/kernel.py, the file will be executed from within its parent directory with the python interpreter +used to run this `compile.py` script +""" + +if __name__ == "__main__": + + # command-line arguments + parser = ArgumentParser(description=desc) + parser.add_argument("path", + help="Path to Python source containing desired kernel in its scope. File will be executed.") + parser.add_argument("--kernel-name", "-n", type=str, default="", help="Name of the kernel to compile", + required=True) + parser.add_argument("--num-warps", "-w", type=int, default=1, help="Number of warps to launch the kernel") + parser.add_argument("--num-stages", "-ns", type=int, default=3, + help="Number of stages (meta-parameter of the kernel)") + parser.add_argument("--out-name", "-on", type=str, default=None, help="Out name for the compiled kernel") + parser.add_argument("--out-path", "-o", type=Path, default=None, help="Out filename") + parser.add_argument("--signature", "-s", type=str, help="Signature of the kernel", required=True) + parser.add_argument("--grid", "-g", type=str, help="Launch grid of the kernel", required=True) + args = parser.parse_args() + + out_name = args.out_name if args.out_name else args.kernel_name + out_path = args.out_path if args.out_path else Path(out_name) + + # execute python sources and extract functions wrapped in JITFunction + arg_path = Path(args.path) + sys.path.insert(0, str(arg_path.parent)) + spec = importlib.util.spec_from_file_location(arg_path.stem, arg_path) + mod = importlib.util.module_from_spec(spec) + spec.loader.exec_module(mod) + kernel = getattr(mod, args.kernel_name) + grid = args.grid.split(",") + assert len(grid) == 3 + + # validate and parse signature + signature = list(map(lambda s: s.strip(" "), args.signature.split(","))) + + def hash_signature(signature: List[str]): + m = hashlib.sha256() + m.update(" ".join(signature).encode()) + return m.hexdigest()[:8] + + meta_sig = f"warps{args.num_warps}xstages{args.num_stages}" + sig_hash = hash_signature(signature + [meta_sig]) + + def constexpr(s): + try: + ret = int(s) + return ret + except ValueError: + pass + try: + ret = float(s) + return ret + except ValueError: + pass + return None + + hints = {i: constexpr(s.split(":")[1]) for i, s in enumerate(signature) if ":" in s} + hints = {k: v for k, v in hints.items() if v is not None} + constants = {kernel.arg_names[i]: constexpr(s) for i, s in enumerate(signature)} + constants = {k: v for k, v in constants.items() if v is not None} + signature = { + kernel.arg_names[i]: s.split(":")[0] + for i, s in enumerate(signature) + if kernel.arg_names[i] not in constants + } + const_sig = 'x'.join([str(v) for v in constants.values()]) + doc_string = [f"{k}={v}" for k, v in constants.items()] + doc_string += [f"num_warps={args.num_warps}", f"num_stages={args.num_stages}"] + + # compile ast into cubin + for h in hints.values(): + assert h in [1, 16], f"Only 1 and 16 are valid hints, got {h}" + attrs = triton.backends.compiler.AttrsDescriptor.from_hints(hints) + for p, v in attrs.get_constants().items(): + constants.update({kernel.arg_names[p]: v}) + src = triton.compiler.ASTSource(fn=kernel, constants=constants, signature=signature, attrs=attrs) + opts = {"num_warps": args.num_warps, "num_stages": args.num_stages} + ccinfo = triton.compile(src, options=opts) + arg_names = [] + arg_types = [] + arg_names_not_1 = [] + arg_types_not_1 = [] + for i, arg_name in enumerate(kernel.arg_names): + if arg_name not in constants: + arg_names.append(arg_name) + arg_types.append(signature[arg_name]) + arg_names_not_1.append(arg_name) + arg_types_not_1.append(signature[arg_name]) + elif i in attrs.equal_to_1: + arg_names.append(arg_name) + arg_types.append(signature[arg_name]) + + # dump C stub code + suffix = kernel_suffix(signature.values(), attrs) + func_name = '_'.join([out_name, sig_hash, suffix]) + hex_ = str(binascii.hexlify(ccinfo.asm["cubin"]))[2:-1] + params = { + "kernel_name": func_name, + "triton_kernel_name": args.kernel_name, + "bin_size": len(hex_), + "bin_data": ", ".join([f"0x{x}{y}" for x, y in zip(hex_[::2], hex_[1::2])]), + "signature": ", ".join([f"{ty_to_cpp(ty)} {name}" for name, ty in zip(arg_names_not_1, arg_types_not_1)]), + "full_signature": ", ".join([f"{ty_to_cpp(ty)} {name}" for name, ty in zip(arg_names, arg_types)]), + "arg_pointers": ", ".join([f"&{arg}" for arg in arg_names_not_1]), + "num_args": len(arg_names_not_1), + "kernel_docstring": doc_string, + "shared": ccinfo.metadata.shared, + "num_warps": args.num_warps, + "algo_info": '_'.join([const_sig, meta_sig]), + "gridX": grid[0], + "gridY": grid[1], + "gridZ": grid[2], + "_placeholder": "", + } + for ext in ['h', 'c']: + template_path = Path(__file__).parent / f"compile.{ext}" + with out_path.with_suffix(f".{sig_hash}_{suffix}.{ext}").open("w") as fp: + fp.write(Path(template_path).read_text().format(**params)) diff --git a/lib/python3.12/site-packages/triton/tools/disasm.py b/lib/python3.12/site-packages/triton/tools/disasm.py new file mode 100644 index 0000000000000000000000000000000000000000..002c4e9b5b59c59c5d6804bc2a19c5a28be9cdfc --- /dev/null +++ b/lib/python3.12/site-packages/triton/tools/disasm.py @@ -0,0 +1,144 @@ +# MIT License + +# Copyright (c) 2020 Da Yan @ HKUST + +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import functools +import os +import re +import subprocess +import tempfile + +FLINE_RE = re.compile(r'\s*/\*\w{4}\*/\s*([^;]*;)\s*/\* 0x(\w{16}) \*/\s*') +SLINE_RE = re.compile(r'\s*/\* 0x(\w{16}) \*/\s*') +FNAME_RE = re.compile(r'\s*Function : (\w+)\s*') +BRA_RE = re.compile(r'(.*BRA(?:\.U)? )(0x\w+);') + + +def parseCtrl(sline): + enc = int(SLINE_RE.match(sline).group(1), 16) + stall = (enc >> 41) & 0xf + yld = (enc >> 45) & 0x1 + wrtdb = (enc >> 46) & 0x7 + readb = (enc >> 49) & 0x7 + watdb = (enc >> 52) & 0x3f + + yld_str = 'Y' if yld == 0 else '-' + wrtdb_str = '-' if wrtdb == 7 else str(wrtdb) + readb_str = '-' if readb == 7 else str(readb) + watdb_str = '--' if watdb == 0 else f'{watdb:02d}' + return f'{watdb_str}:{readb_str}:{wrtdb_str}:{yld_str}:{stall:x}' + + +def processSassLines(fline, sline, labels): + asm = FLINE_RE.match(fline).group(1) + # Remove tailing space + if asm.endswith(" ;"): + asm = asm[:-2] + ";" + ctrl = parseCtrl(sline) + # BRA target address + if BRA_RE.match(asm) is not None: + target = int(BRA_RE.match(asm).group(2), 16) + if target in labels: + pass + else: + labels[target] = len(labels) + return (f'{ctrl}', f'{asm}') + + +@functools.lru_cache() +def get_sass(cubin_asm, fun=None): + fd, path = tempfile.mkstemp() + try: + with open(fd, 'wb') as cubin: + cubin.write(cubin_asm) + sass = extract(path, fun) + finally: + os.remove(path) + return sass + + +@functools.lru_cache() +def path_to_cuobjdump(): + from triton.backends.nvidia.compiler import _path_to_binary + return _path_to_binary("cuobjdump") + + +def extract(file_path, fun): + cuobjdump, _ = path_to_cuobjdump() + if fun is None: + sass_str = subprocess.check_output([cuobjdump, "-sass", file_path]) + else: + sass_str = subprocess.check_output([cuobjdump, "-fun", fun, "-sass", file_path]) + sass_lines = sass_str.splitlines() + line_idx = 0 + while line_idx < len(sass_lines): + line = sass_lines[line_idx].decode() + # format: + # function : + # .headerflags: ... + # /*0000*/ asmstr /*0x...*/ + # /*0x...*/ + + # Looking for new function header (function: ) + while FNAME_RE.match(line) is None: + line_idx += 1 + if line_idx < len(sass_lines): + line = sass_lines[line_idx].decode() + else: + return + + fname = FNAME_RE.match(line).group(1) + ret = '' + ret += f'Function:{fname}\n' + line_idx += 2 # bypass .headerflags + line = sass_lines[line_idx].decode() + # Remapping address to label + labels = {} # address -> label_idx + # store sass asm in buffer and them print them (for labels) + # (ctrl, asm) + asm_buffer = [] + while FLINE_RE.match(line) is not None: + # First line (Offset ASM Encoding) + fline = sass_lines[line_idx].decode() + line_idx += 1 + # Second line (Encoding) + sline = sass_lines[line_idx].decode() + line_idx += 1 + asm_buffer.append(processSassLines(fline, sline, labels)) + # peek the next line + line = sass_lines[line_idx].decode() + # Print sass + # label naming convention: LBB#i + for idx, (ctrl, asm) in enumerate(asm_buffer): + # Print label if this is BRA target + offset = idx * 16 + if offset in labels: + label_name = f'LBB{labels[offset]}' + ret += f'{label_name}:\n' + ret += ctrl + '\t' + # if this is BRA, remap offset to label + if BRA_RE.match(asm): + target = int(BRA_RE.match(asm).group(2), 16) + target_name = f'LBB{labels[target]}' + asm = BRA_RE.sub(rf'\1{target_name};', asm) + ret += asm + '\n' + ret += '\n' + return ret diff --git a/lib/python3.12/site-packages/triton/tools/experimental_descriptor.py b/lib/python3.12/site-packages/triton/tools/experimental_descriptor.py new file mode 100644 index 0000000000000000000000000000000000000000..6077cab6f5fcd1973434911c1be730ed47b1a17f --- /dev/null +++ b/lib/python3.12/site-packages/triton/tools/experimental_descriptor.py @@ -0,0 +1,32 @@ +import torch + +import triton + + +class TmaDescKernelParam: + TMA_DESC_SIZE = 128 + + def __init__(self, ptr, dims, block_dims, element_size): + self.desc = torch.empty(self.TMA_DESC_SIZE, dtype=torch.uint8, device="cpu") + assert len(dims) == len(block_dims) + assert 1 <= len(dims) <= 2 + assert self.desc.data_ptr() % 64 == 0 + + if len(dims) == 1: + triton.runtime.driver.active.utils.fill_1d_tma_descriptor(ptr, dims[0], block_dims[0], element_size, + self.desc.data_ptr()) + else: + triton.runtime.driver.active.utils.fill_2d_tma_descriptor(ptr, dims[0], dims[1], block_dims[0], + block_dims[1], element_size, self.desc.data_ptr()) + + # Return a CUtensorMap* pointer in host memory + def tma_desc_cpu_ptr(self): + return self.desc.data_ptr() + + +def create_1d_tma_descriptor(ptr, dim, block_dim, element_size): + return TmaDescKernelParam(ptr, [dim], [block_dim], element_size) + + +def create_2d_tma_descriptor(ptr, dim1, dim0, block_dim1, block_dim0, element_size): + return TmaDescKernelParam(ptr, [dim1, dim0], [block_dim1, block_dim0], element_size) diff --git a/lib/python3.12/site-packages/triton/tools/link.py b/lib/python3.12/site-packages/triton/tools/link.py new file mode 100644 index 0000000000000000000000000000000000000000..75a1157a52f92bbd5d2eae640af97ea360da2ef3 --- /dev/null +++ b/lib/python3.12/site-packages/triton/tools/link.py @@ -0,0 +1,322 @@ +from collections import defaultdict +from pathlib import Path +from typing import Sequence, Union + +from dataclasses import dataclass + + +def _exists(x): + return x is not None + + +class LinkerError(Exception): + pass + + +@dataclass +class KernelLinkerMeta: + orig_kernel_name: str + arg_names: Sequence[str] + arg_ctypes: Sequence[str] + sizes: Sequence[Union[int, None]] + sig_hash: str + triton_suffix: str + suffix: str + num_specs: int + """ number of specialized arguments """ + + +class HeaderParser: + + def __init__(self) -> None: + import re + + # [kernel_name, c signature] + self.linker_directives = re.compile("//[\\s]*tt-linker:[\\s]*([\\w]+):(.+):(.+)") + # [name, hash, suffix] + self.kernel_name = re.compile("^([\\w]+)_([\\w]+)_([\\w]+)$") + # [(type, name)] + self.c_sig = re.compile("[\\s]*(\\w+)\\s(\\w+)[,]?") + # [d|c] + self.arg_suffix = re.compile("[c,d]") + + self.kernels = defaultdict(list) + + def extract_linker_meta(self, header: str): + for ln in header.splitlines(): + if ln.startswith("//"): + m = self.linker_directives.match(ln) + if _exists(m): + ker_name, c_sig, algo_info = m.group(1), m.group(2), m.group(3) + name, sig_hash, suffix = self._match_name(ker_name) + c_types, arg_names = self._match_c_sig(c_sig) + num_specs, sizes = self._match_suffix(suffix, c_sig) + self._add_kernel( + "_".join([name, algo_info]), + KernelLinkerMeta( + orig_kernel_name=name, + arg_names=arg_names, + arg_ctypes=c_types, + sizes=sizes, + sig_hash=sig_hash, + triton_suffix=suffix, + suffix=suffix, + num_specs=num_specs, + ), + ) + + def _match_name(self, ker_name: str): + m = self.kernel_name.match(ker_name) + if _exists(m): + name, sig_hash, suffix = m.group(1), m.group(2), m.group(3) + return name, sig_hash, suffix + raise LinkerError(f"{ker_name} is not a valid kernel name") + + def _match_c_sig(self, c_sig: str): + m = self.c_sig.findall(c_sig) + if len(m): + tys, args = [], [] + for ty, arg_name in m: + tys.append(ty) + args.append(arg_name) + return tys, args + + raise LinkerError(f"{c_sig} is not a valid argument signature") + + def _match_suffix(self, suffix: str, c_sig: str): + args = c_sig.split(",") + s2i = {"c": 1, "d": 16} + num_specs = 0 + sizes = [] + # scan through suffix, first find the index, + # then see if it is followed by d or c + for i in range(len(args)): + pos = suffix.find(str(i)) + if pos == -1: + raise LinkerError(f"{suffix} is not a valid kernel suffix") + pos += len(str(i)) + if self.arg_suffix.match(suffix, pos): + num_specs += 1 + sizes.extend([None] * (i - len(sizes))) + sizes.append(s2i[suffix[pos]]) + pos += 1 + if i < len(args) - 1: + suffix = suffix[pos:] + else: + sizes.extend([None] * (len(args) - len(sizes))) + return num_specs, sizes + + def _add_kernel(self, name: str, ker: KernelLinkerMeta): + if name in self.kernels: + last: KernelLinkerMeta = self.kernels[name][-1] + + for cur, new_ in zip(last.arg_ctypes, ker.arg_ctypes): + if cur != new_: + raise LinkerError( + f"Mismatched signature for kernel {name}: \n\texisting sig is: {','.join(last.arg_ctypes)}\n\tcurrent is: {','.join(ker.arg_ctypes)}" + ) + + self.kernels[name].append(ker) + + +def gen_signature_with_full_args(m): + return ", ".join([f"{ty} {arg}" for ty, arg in zip(m.arg_ctypes, m.arg_names)]) + + +def gen_signature(m): + arg_types = [ty for ty, hint in zip(m.arg_ctypes, m.sizes) if hint != 1] + arg_names = [arg for arg, hint in zip(m.arg_names, m.sizes) if hint != 1] + sig = ", ".join([f"{ty} {arg}" for ty, arg in zip(arg_types, arg_names)]) + return sig + + +# generate declarations of kernels with meta-parameter and constant values +def make_algo_decls(name: str, metas: Sequence[KernelLinkerMeta]) -> str: + return f""" +CUresult {name}(CUstream stream, {gen_signature_with_full_args(metas[-1])}); +void load_{name}(); +void unload_{name}(); + """ + + +# generate declarations of kernels with meta-parameter and constant values +def make_global_decl(meta: KernelLinkerMeta) -> str: + return f""" +CUresult {meta.orig_kernel_name}_default(CUstream stream, {gen_signature_with_full_args(meta)}); +CUresult {meta.orig_kernel_name}(CUstream stream, {gen_signature_with_full_args(meta)}, int algo_id); +void load_{meta.orig_kernel_name}(); +void unload_{meta.orig_kernel_name}(); + """ + + +# generate dispatcher function for kernels with different meta-parameter and constant values +def make_default_algo_kernel(meta: KernelLinkerMeta) -> str: + src = f"CUresult {meta.orig_kernel_name}_default(CUstream stream, {gen_signature_with_full_args(meta)}){{\n" + src += (f" return {meta.orig_kernel_name}(stream, {', '.join(meta.arg_names)}, 0);\n") + src += "}\n" + return src + + +# generate dispatcher function for kernels with different integer value hints +def make_kernel_hints_dispatcher(name: str, metas: Sequence[KernelLinkerMeta]) -> str: + src = f"// launcher for: {name}\n" + for meta in sorted(metas, key=lambda m: -m.num_specs): + src += f"CUresult {meta.orig_kernel_name}_{meta.sig_hash}_{meta.suffix}(CUstream stream, {gen_signature(meta)});\n" + src += "\n" + + src += (f"CUresult {name}(CUstream stream, {gen_signature_with_full_args(metas[-1])}){{") + src += "\n" + for meta in sorted(metas, key=lambda m: -m.num_specs): + cond_fn = ( # + lambda val, hint: f"({val} % {hint} == 0)" # + if hint == 16 # + else f"({val} == {hint})" # + if hint == 1 # + else None) + conds = " && ".join([ # + cond_fn(val, hint) # + for val, hint in zip(meta.arg_names, meta.sizes) # + if hint is not None + ]) + src += (f" if ({conds})\n" if any(meta.sizes) else "if (1)\n" + ) # Edge case where no specializations hence no dispatching required + arg_names = [arg for arg, hint in zip(meta.arg_names, meta.sizes) if hint != 1] + src += f" return {meta.orig_kernel_name}_{meta.sig_hash}_{meta.suffix}(stream, {', '.join(arg_names)});\n" + src += "\n" + src += " return CUDA_ERROR_INVALID_VALUE;\n" + src += "}\n" + + for mode in ["load", "unload"]: + src += f"\n// {mode} for: {name}\n" + for meta in sorted(metas, key=lambda m: -m.num_specs): + src += f"void {mode}_{meta.orig_kernel_name}_{meta.sig_hash}_{meta.suffix}();\n" + src += f"void {mode}_{name}() {{" + src += "\n" + for meta in sorted(metas, key=lambda m: -m.num_specs): + src += (f" {mode}_{meta.orig_kernel_name}_{meta.sig_hash}_{meta.suffix}();\n") + src += "}\n" + return src + + +# generate dispatcher function for kernels with different meta-parameter and constant values +def make_kernel_meta_const_dispatcher(meta: KernelLinkerMeta) -> str: + src = f"CUresult {meta.orig_kernel_name}(CUstream stream, {gen_signature_with_full_args(meta)}, int algo_id){{\n" + src += f" assert (algo_id < (int)sizeof({meta.orig_kernel_name}_kernels));\n" + src += f" return {meta.orig_kernel_name}_kernels[algo_id](stream, {', '.join(meta.arg_names)});\n" + src += "}\n" + return src + + +# generate definition of function pointers of kernel dispatchers based on meta-parameter and constant values +def make_func_pointers(names: str, meta: KernelLinkerMeta) -> str: + # the table of hint dispatchers + src = f"typedef CUresult (*kernel_func_t)(CUstream stream, {gen_signature_with_full_args(meta)});\n" + src += f"kernel_func_t {meta.orig_kernel_name}_kernels[] = {{\n" + for name in names: + src += f" {name},\n" + src += "};\n" + return src + + +# generate definition for load/unload functions for kernels with different meta-parameter and constant values +def make_kernel_load_def(names: str, meta: KernelLinkerMeta) -> str: + src = "" + for mode in ["load", "unload"]: + src += f"void {mode}_{meta.orig_kernel_name}(void){{\n" + for name in names: + src += f" {mode}_{name}();\n" + src += "}\n\n" + return src + + +def make_get_num_algos_decl(meta: KernelLinkerMeta) -> str: + src = f"int {meta.orig_kernel_name}_get_num_algos(void);" + return src + + +def make_get_num_algos_def(meta: KernelLinkerMeta) -> str: + src = f"int {meta.orig_kernel_name}_get_num_algos(void){{\n" + src += f" return (int)(sizeof({meta.orig_kernel_name}_kernels) / sizeof({meta.orig_kernel_name}_kernels[0]));\n" + src += "}\n" + return src + + +desc = """ +Triton ahead-of-time linker: + +This program takes in header files generated by compile.py, and generates a +single entry-point responsible for dispatching the user's input to the right +kernel given the specializations that were compiled. + +Example usage: +python link.py /path/to/headers/*.h -o kernel_name +""" + +if __name__ == "__main__": + from argparse import ArgumentParser + + parser = ArgumentParser(description=desc) + parser.add_argument( + "headers", + nargs="+", + help="Paths to header files to link. Must include linker directive annotations (autogenerated by ttc)", + ) + parser.add_argument("--out", "-o", type=Path, help="Out filename") + parser.add_argument( + "--prefix", + type=str, + default="", + help="String to prefix kernel dispatcher names", + ) + args = parser.parse_args() + + # metadata + parser = HeaderParser() + includes = [] + for header in args.headers: + h_path = Path(header) + h_str = h_path.read_text() + includes.append(h_path.name) + parser.extract_linker_meta(h_str) + + # generate headers + algo_decls = [make_algo_decls(name, meta) for name, meta in parser.kernels.items()] + meta_lists = [meta for name, meta in parser.kernels.items()] + meta = meta_lists[0][0] + get_num_algos_decl = make_get_num_algos_decl(meta) + global_decl = make_global_decl(meta) + with args.out.with_suffix(".h").open("w") as fp: + out = "#include \n" + out += "\n".join(algo_decls) + out += "\n" + out += get_num_algos_decl + out += "\n" + out += global_decl + fp.write(out) + + # generate source + defs = [make_kernel_hints_dispatcher(name, meta) for name, meta in parser.kernels.items()] + names = [name for name in parser.kernels.keys()] + func_pointers_def = make_func_pointers(names, meta) + meta_const_def = make_kernel_meta_const_dispatcher(meta) + load_unload_def = make_kernel_load_def(names, meta) + get_num_algos_def = make_get_num_algos_def(meta) + default_algo_kernel = make_default_algo_kernel(meta) + with args.out.with_suffix(".c").open("w") as fp: + out = "" + out += "#include \n" + out += "#include \n" + out += "#include \n" + out += "\n" + out += "\n".join(defs) + out += "\n" + out += func_pointers_def + out += "\n" + out += get_num_algos_def + out += "\n" + out += meta_const_def + out += "\n" + out += load_unload_def + out += "\n" + out += default_algo_kernel + fp.write(out) diff --git a/lib/python3.12/site-packages/typing_inspection/__init__.py b/lib/python3.12/site-packages/typing_inspection/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/typing_inspection/__pycache__/__init__.cpython-312.pyc b/lib/python3.12/site-packages/typing_inspection/__pycache__/__init__.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..cafddc8925493046edc3936a34fe67ad11281736 Binary files /dev/null and b/lib/python3.12/site-packages/typing_inspection/__pycache__/__init__.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/typing_inspection/__pycache__/introspection.cpython-312.pyc b/lib/python3.12/site-packages/typing_inspection/__pycache__/introspection.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..324156709d2afb98b1d16008b0ab287730b8efc6 Binary files /dev/null and b/lib/python3.12/site-packages/typing_inspection/__pycache__/introspection.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/typing_inspection/__pycache__/typing_objects.cpython-312.pyc b/lib/python3.12/site-packages/typing_inspection/__pycache__/typing_objects.cpython-312.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ec380d3f80c6eba27c65349327b8abd4408af730 Binary files /dev/null and b/lib/python3.12/site-packages/typing_inspection/__pycache__/typing_objects.cpython-312.pyc differ diff --git a/lib/python3.12/site-packages/typing_inspection/introspection.py b/lib/python3.12/site-packages/typing_inspection/introspection.py new file mode 100644 index 0000000000000000000000000000000000000000..d6c083e5456c8ad5da7559a3c0a901e0812a5014 --- /dev/null +++ b/lib/python3.12/site-packages/typing_inspection/introspection.py @@ -0,0 +1,587 @@ +"""High-level introspection utilities, used to inspect type annotations.""" + +from __future__ import annotations + +import sys +import types +from collections.abc import Generator +from dataclasses import InitVar +from enum import Enum, IntEnum, auto +from typing import Any, Literal, NamedTuple, cast + +from typing_extensions import TypeAlias, assert_never, get_args, get_origin + +from . import typing_objects + +__all__ = ( + 'AnnotationSource', + 'ForbiddenQualifier', + 'InspectedAnnotation', + 'Qualifier', + 'get_literal_values', + 'inspect_annotation', + 'is_union_origin', +) + +if sys.version_info >= (3, 14) or sys.version_info < (3, 10): + + def is_union_origin(obj: Any, /) -> bool: + """Return whether the provided origin is the union form. + + ```pycon + >>> is_union_origin(typing.Union) + True + >>> is_union_origin(get_origin(int | str)) + True + >>> is_union_origin(types.UnionType) + True + ``` + + !!! note + Since Python 3.14, both `Union[, , ...]` and ` | | ...` forms create instances + of the same [`typing.Union`][] class. As such, it is recommended to not use this function + anymore (provided that you only support Python 3.14 or greater), and instead use the + [`typing_objects.is_union()`][typing_inspection.typing_objects.is_union] function directly: + + ```python + from typing import Union, get_origin + + from typing_inspection import typing_objects + + typ = int | str # Or Union[int, str] + origin = get_origin(typ) + if typing_objects.is_union(origin): + ... + ``` + """ + return typing_objects.is_union(obj) + + +else: + + def is_union_origin(obj: Any, /) -> bool: + """Return whether the provided origin is the union form. + + ```pycon + >>> is_union_origin(typing.Union) + True + >>> is_union_origin(get_origin(int | str)) + True + >>> is_union_origin(types.UnionType) + True + ``` + + !!! note + Since Python 3.14, both `Union[, , ...]` and ` | | ...` forms create instances + of the same [`typing.Union`][] class. As such, it is recommended to not use this function + anymore (provided that you only support Python 3.14 or greater), and instead use the + [`typing_objects.is_union()`][typing_inspection.typing_objects.is_union] function directly: + + ```python + from typing import Union, get_origin + + from typing_inspection import typing_objects + + typ = int | str # Or Union[int, str] + origin = get_origin(typ) + if typing_objects.is_union(origin): + ... + ``` + """ + return typing_objects.is_union(obj) or obj is types.UnionType + + +def _literal_type_check(value: Any, /) -> None: + """Type check the provided literal value against the legal parameters.""" + if ( + not isinstance(value, (int, bytes, str, bool, Enum, typing_objects.NoneType)) + and value is not typing_objects.NoneType + ): + raise TypeError(f'{value} is not a valid literal value, must be one of: int, bytes, str, Enum, None.') + + +def get_literal_values( + annotation: Any, + /, + *, + type_check: bool = False, + unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'eager', +) -> Generator[Any]: + """Yield the values contained in the provided [`Literal`][typing.Literal] [special form][]. + + Args: + annotation: The [`Literal`][typing.Literal] [special form][] to unpack. + type_check: Whether to check if the literal values are [legal parameters][literal-legal-parameters]. + Raises a [`TypeError`][] otherwise. + unpack_type_aliases: What to do when encountering [PEP 695](https://peps.python.org/pep-0695/) + [type aliases][type-aliases]. Can be one of: + + - `'skip'`: Do not try to parse type aliases. Note that this can lead to incorrect results: + ```pycon + >>> type MyAlias = Literal[1, 2] + >>> list(get_literal_values(Literal[MyAlias, 3], unpack_type_aliases="skip")) + [MyAlias, 3] + ``` + + - `'lenient'`: Try to parse type aliases, and fallback to `'skip'` if the type alias can't be inspected + (because of an undefined forward reference). + + - `'eager'`: Parse type aliases and raise any encountered [`NameError`][] exceptions (the default): + ```pycon + >>> type MyAlias = Literal[1, 2] + >>> list(get_literal_values(Literal[MyAlias, 3], unpack_type_aliases="eager")) + [1, 2, 3] + ``` + + Note: + While `None` is [equivalent to][none] `type(None)`, the runtime implementation of [`Literal`][typing.Literal] + does not de-duplicate them. This function makes sure this de-duplication is applied: + + ```pycon + >>> list(get_literal_values(Literal[NoneType, None])) + [None] + ``` + + Example: + ```pycon + >>> type Ints = Literal[1, 2] + >>> list(get_literal_values(Literal[1, Ints], unpack_type_alias="skip")) + ["a", Ints] + >>> list(get_literal_values(Literal[1, Ints])) + [1, 2] + >>> list(get_literal_values(Literal[1.0], type_check=True)) + Traceback (most recent call last): + ... + TypeError: 1.0 is not a valid literal value, must be one of: int, bytes, str, Enum, None. + ``` + """ + # `literal` is guaranteed to be a `Literal[...]` special form, so use + # `__args__` directly instead of calling `get_args()`. + + if unpack_type_aliases == 'skip': + _has_none = False + # `Literal` parameters are already deduplicated, no need to do it ourselves. + # (we only check for `None` and `NoneType`, which should be considered as duplicates). + for arg in annotation.__args__: + if type_check: + _literal_type_check(arg) + if arg is None or arg is typing_objects.NoneType: + if not _has_none: + yield None + _has_none = True + else: + yield arg + else: + # We'll need to manually deduplicate parameters, see the `Literal` implementation in `typing`. + values_and_type: list[tuple[Any, type[Any]]] = [] + + for arg in annotation.__args__: + # Note: we could also check for generic aliases with a type alias as an origin. + # However, it is very unlikely that this happens as type variables can't appear in + # `Literal` forms, so the only valid (but unnecessary) use case would be something like: + # `type Test[T] = Literal['a']` (and then use `Test[SomeType]`). + if typing_objects.is_typealiastype(arg): + try: + alias_value = arg.__value__ + except NameError: + if unpack_type_aliases == 'eager': + raise + # unpack_type_aliases == "lenient": + if type_check: + _literal_type_check(arg) + values_and_type.append((arg, type(arg))) + else: + sub_args = get_literal_values( + alias_value, type_check=type_check, unpack_type_aliases=unpack_type_aliases + ) + values_and_type.extend((a, type(a)) for a in sub_args) # pyright: ignore[reportUnknownArgumentType] + else: + if type_check: + _literal_type_check(arg) + if arg is typing_objects.NoneType: + values_and_type.append((None, typing_objects.NoneType)) + else: + values_and_type.append((arg, type(arg))) # pyright: ignore[reportUnknownArgumentType] + + try: + dct = dict.fromkeys(values_and_type) + except TypeError: + # Unhashable parameters, the Python implementation allows them + yield from (p for p, _ in values_and_type) + else: + yield from (p for p, _ in dct) + + +Qualifier: TypeAlias = Literal['required', 'not_required', 'read_only', 'class_var', 'init_var', 'final'] +"""A [type qualifier][].""" + +_all_qualifiers: set[Qualifier] = set(get_args(Qualifier)) + + +# TODO at some point, we could switch to an enum flag, so that multiple sources +# can be combined. However, is there a need for this? +class AnnotationSource(IntEnum): + # TODO if/when https://peps.python.org/pep-0767/ is accepted, add 'read_only' + # to CLASS and NAMED_TUPLE (even though for named tuples it is redundant). + + """The source of an annotation, e.g. a class or a function. + + Depending on the source, different [type qualifiers][type qualifier] may be (dis)allowed. + """ + + ASSIGNMENT_OR_VARIABLE = auto() + """An annotation used in an assignment or variable annotation: + + ```python + x: Final[int] = 1 + y: Final[str] + ``` + + **Allowed type qualifiers:** [`Final`][typing.Final]. + """ + + CLASS = auto() + """An annotation used in the body of a class: + + ```python + class Test: + x: Final[int] = 1 + y: ClassVar[str] + ``` + + **Allowed type qualifiers:** [`ClassVar`][typing.ClassVar], [`Final`][typing.Final]. + """ + + DATACLASS = auto() + """An annotation used in the body of a dataclass: + + ```python + @dataclass + class Test: + x: Final[int] = 1 + y: InitVar[str] = 'test' + ``` + + **Allowed type qualifiers:** [`ClassVar`][typing.ClassVar], [`Final`][typing.Final], [`InitVar`][dataclasses.InitVar]. + """ # noqa: E501 + + TYPED_DICT = auto() + """An annotation used in the body of a [`TypedDict`][typing.TypedDict]: + + ```python + class TD(TypedDict): + x: Required[ReadOnly[int]] + y: ReadOnly[NotRequired[str]] + ``` + + **Allowed type qualifiers:** [`ReadOnly`][typing.ReadOnly], [`Required`][typing.Required], + [`NotRequired`][typing.NotRequired]. + """ + + NAMED_TUPLE = auto() + """An annotation used in the body of a [`NamedTuple`][typing.NamedTuple]. + + ```python + class NT(NamedTuple): + x: int + y: str + ``` + + **Allowed type qualifiers:** none. + """ + + FUNCTION = auto() + """An annotation used in a function, either for a parameter or the return value. + + ```python + def func(a: int) -> str: + ... + ``` + + **Allowed type qualifiers:** none. + """ + + ANY = auto() + """An annotation that might come from any source. + + **Allowed type qualifiers:** all. + """ + + BARE = auto() + """An annotation that is inspected as is. + + **Allowed type qualifiers:** none. + """ + + @property + def allowed_qualifiers(self) -> set[Qualifier]: + """The allowed [type qualifiers][type qualifier] for this annotation source.""" + # TODO use a match statement when Python 3.9 support is dropped. + if self is AnnotationSource.ASSIGNMENT_OR_VARIABLE: + return {'final'} + elif self is AnnotationSource.CLASS: + return {'final', 'class_var'} + elif self is AnnotationSource.DATACLASS: + return {'final', 'class_var', 'init_var'} + elif self is AnnotationSource.TYPED_DICT: + return {'required', 'not_required', 'read_only'} + elif self in (AnnotationSource.NAMED_TUPLE, AnnotationSource.FUNCTION, AnnotationSource.BARE): + return set() + elif self is AnnotationSource.ANY: + return _all_qualifiers + else: # pragma: no cover + assert_never(self) + + +class ForbiddenQualifier(Exception): + """The provided [type qualifier][] is forbidden.""" + + qualifier: Qualifier + """The forbidden qualifier.""" + + def __init__(self, qualifier: Qualifier, /) -> None: + self.qualifier = qualifier + + +class _UnknownTypeEnum(Enum): + UNKNOWN = auto() + + def __str__(self) -> str: + return 'UNKNOWN' + + def __repr__(self) -> str: + return '' + + +UNKNOWN = _UnknownTypeEnum.UNKNOWN +"""A sentinel value used when no [type expression][] is present.""" + +_UnkownType: TypeAlias = Literal[_UnknownTypeEnum.UNKNOWN] +"""The type of the [`UNKNOWN`][typing_inspection.introspection.UNKNOWN] sentinel value.""" + + +class InspectedAnnotation(NamedTuple): + """The result of the inspected annotation.""" + + type: Any | _UnkownType + """The final [type expression][], with [type qualifiers][type qualifier] and annotated metadata stripped. + + If no type expression is available, the [`UNKNOWN`][typing_inspection.introspection.UNKNOWN] sentinel + value is used instead. This is the case when a [type qualifier][] is used with no type annotation: + + ```python + ID: Final = 1 + + class C: + x: ClassVar = 'test' + ``` + """ + + qualifiers: set[Qualifier] + """The [type qualifiers][type qualifier] present on the annotation.""" + + metadata: list[Any] + """The annotated metadata.""" + + +def inspect_annotation( # noqa: PLR0915 + annotation: Any, + /, + *, + annotation_source: AnnotationSource, + unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'skip', +) -> InspectedAnnotation: + """Inspect an [annotation expression][], extracting any [type qualifier][] and metadata. + + An [annotation expression][] is a [type expression][] optionally surrounded by one or more + [type qualifiers][type qualifier] or by [`Annotated`][typing.Annotated]. This function will: + + - Unwrap the type expression, keeping track of the type qualifiers. + - Unwrap [`Annotated`][typing.Annotated] forms, keeping track of the annotated metadata. + + Args: + annotation: The annotation expression to be inspected. + annotation_source: The source of the annotation. Depending on the source (e.g. a class), different type + qualifiers may be (dis)allowed. To allow any type qualifier, use + [`AnnotationSource.ANY`][typing_inspection.introspection.AnnotationSource.ANY]. + unpack_type_aliases: What to do when encountering [PEP 695](https://peps.python.org/pep-0695/) + [type aliases][type-aliases]. Can be one of: + + - `'skip'`: Do not try to parse type aliases (the default): + ```pycon + >>> type MyInt = Annotated[int, 'meta'] + >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='skip') + InspectedAnnotation(type=MyInt, qualifiers={}, metadata=[]) + ``` + + - `'lenient'`: Try to parse type aliases, and fallback to `'skip'` if the type alias + can't be inspected (because of an undefined forward reference): + ```pycon + >>> type MyInt = Annotated[Undefined, 'meta'] + >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='lenient') + InspectedAnnotation(type=MyInt, qualifiers={}, metadata=[]) + >>> Undefined = int + >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='lenient') + InspectedAnnotation(type=int, qualifiers={}, metadata=['meta']) + ``` + + - `'eager'`: Parse type aliases and raise any encountered [`NameError`][] exceptions. + + Returns: + The result of the inspected annotation, where the type expression, used qualifiers and metadata is stored. + + Example: + ```pycon + >>> inspect_annotation( + ... Final[Annotated[ClassVar[Annotated[int, 'meta_1']], 'meta_2']], + ... annotation_source=AnnotationSource.CLASS, + ... ) + ... + InspectedAnnotation(type=int, qualifiers={'class_var', 'final'}, metadata=['meta_1', 'meta_2']) + ``` + """ + allowed_qualifiers = annotation_source.allowed_qualifiers + qualifiers: set[Qualifier] = set() + metadata: list[Any] = [] + + while True: + annotation, _meta = _unpack_annotated(annotation, unpack_type_aliases=unpack_type_aliases) + if _meta: + metadata = _meta + metadata + continue + + origin = get_origin(annotation) + if origin is not None: + if typing_objects.is_classvar(origin): + if 'class_var' not in allowed_qualifiers: + raise ForbiddenQualifier('class_var') + qualifiers.add('class_var') + annotation = annotation.__args__[0] + elif typing_objects.is_final(origin): + if 'final' not in allowed_qualifiers: + raise ForbiddenQualifier('final') + qualifiers.add('final') + annotation = annotation.__args__[0] + elif typing_objects.is_required(origin): + if 'required' not in allowed_qualifiers: + raise ForbiddenQualifier('required') + qualifiers.add('required') + annotation = annotation.__args__[0] + elif typing_objects.is_notrequired(origin): + if 'not_required' not in allowed_qualifiers: + raise ForbiddenQualifier('not_required') + qualifiers.add('not_required') + annotation = annotation.__args__[0] + elif typing_objects.is_readonly(origin): + if 'read_only' not in allowed_qualifiers: + raise ForbiddenQualifier('not_required') + qualifiers.add('read_only') + annotation = annotation.__args__[0] + else: + # origin is not None but not a type qualifier nor `Annotated` (e.g. `list[int]`): + break + elif isinstance(annotation, InitVar): + if 'init_var' not in allowed_qualifiers: + raise ForbiddenQualifier('init_var') + qualifiers.add('init_var') + annotation = cast(Any, annotation.type) + else: + break + + # `Final`, `ClassVar` and `InitVar` are type qualifiers allowed to be used as a bare annotation: + if typing_objects.is_final(annotation): + if 'final' not in allowed_qualifiers: + raise ForbiddenQualifier('final') + qualifiers.add('final') + annotation = UNKNOWN + elif typing_objects.is_classvar(annotation): + if 'class_var' not in allowed_qualifiers: + raise ForbiddenQualifier('class_var') + qualifiers.add('class_var') + annotation = UNKNOWN + elif annotation is InitVar: + if 'init_var' not in allowed_qualifiers: + raise ForbiddenQualifier('init_var') + qualifiers.add('init_var') + annotation = UNKNOWN + + return InspectedAnnotation(annotation, qualifiers, metadata) + + +def _unpack_annotated_inner( + annotation: Any, unpack_type_aliases: Literal['lenient', 'eager'], check_annotated: bool +) -> tuple[Any, list[Any]]: + origin = get_origin(annotation) + if check_annotated and typing_objects.is_annotated(origin): + annotated_type = annotation.__origin__ + metadata = list(annotation.__metadata__) + + # The annotated type might be a PEP 695 type alias, so we need to recursively + # unpack it. Because Python already flattens `Annotated[Annotated[, ...], ...]` forms, + # we can skip the `is_annotated()` check in the next call: + annotated_type, sub_meta = _unpack_annotated_inner( + annotated_type, unpack_type_aliases=unpack_type_aliases, check_annotated=False + ) + metadata = sub_meta + metadata + return annotated_type, metadata + elif typing_objects.is_typealiastype(annotation): + try: + value = annotation.__value__ + except NameError: + if unpack_type_aliases == 'eager': + raise + else: + typ, metadata = _unpack_annotated_inner( + value, unpack_type_aliases=unpack_type_aliases, check_annotated=True + ) + if metadata: + # Having metadata means the type alias' `__value__` was an `Annotated` form + # (or, recursively, a type alias to an `Annotated` form). It is important to check + # for this, as we don't want to unpack other type aliases (e.g. `type MyInt = int`). + return typ, metadata + return annotation, [] + elif typing_objects.is_typealiastype(origin): + # When parameterized, PEP 695 type aliases become generic aliases + # (e.g. with `type MyList[T] = Annotated[list[T], ...]`, `MyList[int]` + # is a generic alias). + try: + value = origin.__value__ + except NameError: + if unpack_type_aliases == 'eager': + raise + else: + # While Python already handles type variable replacement for simple `Annotated` forms, + # we need to manually apply the same logic for PEP 695 type aliases: + # - With `MyList = Annotated[list[T], ...]`, `MyList[int] == Annotated[list[int], ...]` + # - With `type MyList[T] = Annotated[list[T], ...]`, `MyList[int].__value__ == Annotated[list[T], ...]`. + + try: + # To do so, we emulate the parameterization of the value with the arguments: + # with `type MyList[T] = Annotated[list[T], ...]`, to emulate `MyList[int]`, + # we do `Annotated[list[T], ...][int]` (which gives `Annotated[list[T], ...]`): + value = value[annotation.__args__] + except TypeError: + # Might happen if the type alias is parameterized, but its value doesn't have any + # type variables, e.g. `type MyInt[T] = int`. + pass + typ, metadata = _unpack_annotated_inner( + value, unpack_type_aliases=unpack_type_aliases, check_annotated=True + ) + if metadata: + return typ, metadata + return annotation, [] + + return annotation, [] + + +# This could eventually be made public: +def _unpack_annotated( + annotation: Any, /, *, unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'eager' +) -> tuple[Any, list[Any]]: + if unpack_type_aliases == 'skip': + if typing_objects.is_annotated(get_origin(annotation)): + return annotation.__origin__, list(annotation.__metadata__) + else: + return annotation, [] + + return _unpack_annotated_inner(annotation, unpack_type_aliases=unpack_type_aliases, check_annotated=True) diff --git a/lib/python3.12/site-packages/typing_inspection/py.typed b/lib/python3.12/site-packages/typing_inspection/py.typed new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/lib/python3.12/site-packages/typing_inspection/typing_objects.py b/lib/python3.12/site-packages/typing_inspection/typing_objects.py new file mode 100644 index 0000000000000000000000000000000000000000..dc44ba98cd8ccdba374982819083d89bbf808c2e --- /dev/null +++ b/lib/python3.12/site-packages/typing_inspection/typing_objects.py @@ -0,0 +1,607 @@ +"""Low-level introspection utilities for [`typing`][] members. + +The provided functions in this module check against both the [`typing`][] and [`typing_extensions`][] +variants, if they exists and are different. +""" +# ruff: noqa: UP006 + +import collections.abc +import contextlib +import re +import sys +import typing +import warnings +from textwrap import dedent +from types import FunctionType, GenericAlias +from typing import Any, Final + +import typing_extensions +from typing_extensions import LiteralString, TypeAliasType, TypeIs, deprecated + +__all__ = ( + 'DEPRECATED_ALIASES', + 'NoneType', + 'is_annotated', + 'is_any', + 'is_classvar', + 'is_concatenate', + 'is_deprecated', + 'is_final', + 'is_forwardref', + 'is_generic', + 'is_literal', + 'is_literalstring', + 'is_namedtuple', + 'is_never', + 'is_newtype', + 'is_nodefault', + 'is_noextraitems', + 'is_noreturn', + 'is_notrequired', + 'is_paramspec', + 'is_paramspecargs', + 'is_paramspeckwargs', + 'is_readonly', + 'is_required', + 'is_self', + 'is_typealias', + 'is_typealiastype', + 'is_typeguard', + 'is_typeis', + 'is_typevar', + 'is_typevartuple', + 'is_union', + 'is_unpack', +) + +_IS_PY310 = sys.version_info[:2] == (3, 10) + + +def _compile_identity_check_function(member: LiteralString, function_name: LiteralString) -> FunctionType: + """Create a function checking that the function argument is the (unparameterized) typing `member`. + + The function will make sure to check against both the `typing` and `typing_extensions` + variants as depending on the Python version, the `typing_extensions` variant might be different. + For instance, on Python 3.9: + + ```pycon + >>> from typing import Literal as t_Literal + >>> from typing_extensions import Literal as te_Literal, get_origin + + >>> t_Literal is te_Literal + False + >>> get_origin(t_Literal[1]) + typing.Literal + >>> get_origin(te_Literal[1]) + typing_extensions.Literal + ``` + """ + in_typing = hasattr(typing, member) + in_typing_extensions = hasattr(typing_extensions, member) + + if in_typing and in_typing_extensions: + if getattr(typing, member) is getattr(typing_extensions, member): + check_code = f'obj is typing.{member}' + else: + check_code = f'obj is typing.{member} or obj is typing_extensions.{member}' + elif in_typing and not in_typing_extensions: + check_code = f'obj is typing.{member}' + elif not in_typing and in_typing_extensions: + check_code = f'obj is typing_extensions.{member}' + else: + check_code = 'False' + + func_code = dedent(f""" + def {function_name}(obj: Any, /) -> bool: + return {check_code} + """) + + locals_: dict[str, Any] = {} + globals_: dict[str, Any] = {'Any': Any, 'typing': typing, 'typing_extensions': typing_extensions} + exec(func_code, globals_, locals_) + return locals_[function_name] + + +def _compile_isinstance_check_function(member: LiteralString, function_name: LiteralString) -> FunctionType: + """Create a function checking that the function is an instance of the typing `member`. + + The function will make sure to check against both the `typing` and `typing_extensions` + variants as depending on the Python version, the `typing_extensions` variant might be different. + """ + in_typing = hasattr(typing, member) + in_typing_extensions = hasattr(typing_extensions, member) + + if in_typing and in_typing_extensions: + if getattr(typing, member) is getattr(typing_extensions, member): + check_code = f'isinstance(obj, typing.{member})' + else: + check_code = f'isinstance(obj, (typing.{member}, typing_extensions.{member}))' + elif in_typing and not in_typing_extensions: + check_code = f'isinstance(obj, typing.{member})' + elif not in_typing and in_typing_extensions: + check_code = f'isinstance(obj, typing_extensions.{member})' + else: + check_code = 'False' + + func_code = dedent(f""" + def {function_name}(obj: Any, /) -> 'TypeIs[{member}]': + return {check_code} + """) + + locals_: dict[str, Any] = {} + globals_: dict[str, Any] = {'Any': Any, 'typing': typing, 'typing_extensions': typing_extensions} + exec(func_code, globals_, locals_) + return locals_[function_name] + + +if sys.version_info >= (3, 10): + from types import NoneType +else: + NoneType = type(None) + +# Keep this ordered, as per `typing.__all__`: + +is_annotated = _compile_identity_check_function('Annotated', 'is_annotated') +is_annotated.__doc__ = """ +Return whether the argument is the [`Annotated`][typing.Annotated] [special form][]. + +```pycon +>>> is_annotated(Annotated) +True +>>> is_annotated(Annotated[int, ...]) +False +``` +""" + +is_any = _compile_identity_check_function('Any', 'is_any') +is_any.__doc__ = """ +Return whether the argument is the [`Any`][typing.Any] [special form][]. + +```pycon +>>> is_any(Any) +True +``` +""" + +is_classvar = _compile_identity_check_function('ClassVar', 'is_classvar') +is_classvar.__doc__ = """ +Return whether the argument is the [`ClassVar`][typing.ClassVar] [type qualifier][]. + +```pycon +>>> is_classvar(ClassVar) +True +>>> is_classvar(ClassVar[int]) +>>> False +``` +""" + +is_concatenate = _compile_identity_check_function('Concatenate', 'is_concatenate') +is_concatenate.__doc__ = """ +Return whether the argument is the [`Concatenate`][typing.Concatenate] [special form][]. + +```pycon +>>> is_concatenate(Concatenate) +True +>>> is_concatenate(Concatenate[int, P]) +False +``` +""" + +is_final = _compile_identity_check_function('Final', 'is_final') +is_final.__doc__ = """ +Return whether the argument is the [`Final`][typing.Final] [type qualifier][]. + +```pycon +>>> is_final(Final) +True +>>> is_final(Final[int]) +False +``` +""" + + +# Unlikely to have a different version in `typing-extensions`, but keep it consistent. +# Also note that starting in 3.14, this is an alias to `annotationlib.ForwardRef`, but +# accessing it from `typing` doesn't seem to be deprecated. +is_forwardref = _compile_isinstance_check_function('ForwardRef', 'is_forwardref') +is_forwardref.__doc__ = """ +Return whether the argument is an instance of [`ForwardRef`][typing.ForwardRef]. + +```pycon +>>> is_forwardref(ForwardRef('T')) +True +``` +""" + + +is_generic = _compile_identity_check_function('Generic', 'is_generic') +is_generic.__doc__ = """ +Return whether the argument is the [`Generic`][typing.Generic] [special form][]. + +```pycon +>>> is_generic(Generic) +True +>>> is_generic(Generic[T]) +False +``` +""" + +is_literal = _compile_identity_check_function('Literal', 'is_literal') +is_literal.__doc__ = """ +Return whether the argument is the [`Literal`][typing.Literal] [special form][]. + +```pycon +>>> is_literal(Literal) +True +>>> is_literal(Literal["a"]) +False +``` +""" + + +# `get_origin(Optional[int]) is Union`, so `is_optional()` isn't implemented. + +is_paramspec = _compile_isinstance_check_function('ParamSpec', 'is_paramspec') +is_paramspec.__doc__ = """ +Return whether the argument is an instance of [`ParamSpec`][typing.ParamSpec]. + +```pycon +>>> P = ParamSpec('P') +>>> is_paramspec(P) +True +``` +""" + +# Protocol? + +is_typevar = _compile_isinstance_check_function('TypeVar', 'is_typevar') +is_typevar.__doc__ = """ +Return whether the argument is an instance of [`TypeVar`][typing.TypeVar]. + +```pycon +>>> T = TypeVar('T') +>>> is_typevar(T) +True +``` +""" + +is_typevartuple = _compile_isinstance_check_function('TypeVarTuple', 'is_typevartuple') +is_typevartuple.__doc__ = """ +Return whether the argument is an instance of [`TypeVarTuple`][typing.TypeVarTuple]. + +```pycon +>>> Ts = TypeVarTuple('Ts') +>>> is_typevartuple(Ts) +True +``` +""" + +is_union = _compile_identity_check_function('Union', 'is_union') +is_union.__doc__ = """ +Return whether the argument is the [`Union`][typing.Union] [special form][]. + +This function can also be used to check for the [`Optional`][typing.Optional] [special form][], +as at runtime, `Optional[int]` is equivalent to `Union[int, None]`. + +```pycon +>>> is_union(Union) +True +>>> is_union(Union[int, str]) +False +``` + +!!! warning + This does not check for unions using the [new syntax][types-union] (e.g. `int | str`). +""" + + +def is_namedtuple(obj: Any, /) -> bool: + """Return whether the argument is a named tuple type. + + This includes [`NamedTuple`][typing.NamedTuple] subclasses and classes created from the + [`collections.namedtuple`][] factory function. + + ```pycon + >>> class User(NamedTuple): + ... name: str + ... + >>> is_namedtuple(User) + True + >>> City = collections.namedtuple('City', []) + >>> is_namedtuple(City) + True + >>> is_namedtuple(NamedTuple) + False + ``` + """ + return isinstance(obj, type) and issubclass(obj, tuple) and hasattr(obj, '_fields') # pyright: ignore[reportUnknownArgumentType] + + +# TypedDict? + +# BinaryIO? IO? TextIO? + +is_literalstring = _compile_identity_check_function('LiteralString', 'is_literalstring') +is_literalstring.__doc__ = """ +Return whether the argument is the [`LiteralString`][typing.LiteralString] [special form][]. + +```pycon +>>> is_literalstring(LiteralString) +True +``` +""" + +is_never = _compile_identity_check_function('Never', 'is_never') +is_never.__doc__ = """ +Return whether the argument is the [`Never`][typing.Never] [special form][]. + +```pycon +>>> is_never(Never) +True +``` +""" + +if sys.version_info >= (3, 10): + is_newtype = _compile_isinstance_check_function('NewType', 'is_newtype') +else: # On Python 3.10, `NewType` is a function. + + def is_newtype(obj: Any, /) -> bool: + return hasattr(obj, '__supertype__') + + +is_newtype.__doc__ = """ +Return whether the argument is a [`NewType`][typing.NewType]. + +```pycon +>>> UserId = NewType("UserId", int) +>>> is_newtype(UserId) +True +``` +""" + +is_nodefault = _compile_identity_check_function('NoDefault', 'is_nodefault') +is_nodefault.__doc__ = """ +Return whether the argument is the [`NoDefault`][typing.NoDefault] sentinel object. + +```pycon +>>> is_nodefault(NoDefault) +True +``` +""" + +is_noextraitems = _compile_identity_check_function('NoExtraItems', 'is_noextraitems') +is_noextraitems.__doc__ = """ +Return whether the argument is the `NoExtraItems` sentinel object. + +```pycon +>>> is_noextraitems(NoExtraItems) +True +``` +""" + +is_noreturn = _compile_identity_check_function('NoReturn', 'is_noreturn') +is_noreturn.__doc__ = """ +Return whether the argument is the [`NoReturn`][typing.NoReturn] [special form][]. + +```pycon +>>> is_noreturn(NoReturn) +True +>>> is_noreturn(Never) +False +``` +""" + +is_notrequired = _compile_identity_check_function('NotRequired', 'is_notrequired') +is_notrequired.__doc__ = """ +Return whether the argument is the [`NotRequired`][typing.NotRequired] [special form][]. + +```pycon +>>> is_notrequired(NotRequired) +True +``` +""" + +is_paramspecargs = _compile_isinstance_check_function('ParamSpecArgs', 'is_paramspecargs') +is_paramspecargs.__doc__ = """ +Return whether the argument is an instance of [`ParamSpecArgs`][typing.ParamSpecArgs]. + +```pycon +>>> P = ParamSpec('P') +>>> is_paramspecargs(P.args) +True +``` +""" + +is_paramspeckwargs = _compile_isinstance_check_function('ParamSpecKwargs', 'is_paramspeckwargs') +is_paramspeckwargs.__doc__ = """ +Return whether the argument is an instance of [`ParamSpecKwargs`][typing.ParamSpecKwargs]. + +```pycon +>>> P = ParamSpec('P') +>>> is_paramspeckwargs(P.kwargs) +True +``` +""" + +is_readonly = _compile_identity_check_function('ReadOnly', 'is_readonly') +is_readonly.__doc__ = """ +Return whether the argument is the [`ReadOnly`][typing.ReadOnly] [special form][]. + +```pycon +>>> is_readonly(ReadOnly) +True +``` +""" + +is_required = _compile_identity_check_function('Required', 'is_required') +is_required.__doc__ = """ +Return whether the argument is the [`Required`][typing.Required] [special form][]. + +```pycon +>>> is_required(Required) +True +``` +""" + +is_self = _compile_identity_check_function('Self', 'is_self') +is_self.__doc__ = """ +Return whether the argument is the [`Self`][typing.Self] [special form][]. + +```pycon +>>> is_self(Self) +True +``` +""" + +# TYPE_CHECKING? + +is_typealias = _compile_identity_check_function('TypeAlias', 'is_typealias') +is_typealias.__doc__ = """ +Return whether the argument is the [`TypeAlias`][typing.TypeAlias] [special form][]. + +```pycon +>>> is_typealias(TypeAlias) +True +``` +""" + +is_typeguard = _compile_identity_check_function('TypeGuard', 'is_typeguard') +is_typeguard.__doc__ = """ +Return whether the argument is the [`TypeGuard`][typing.TypeGuard] [special form][]. + +```pycon +>>> is_typeguard(TypeGuard) +True +``` +""" + +is_typeis = _compile_identity_check_function('TypeIs', 'is_typeis') +is_typeis.__doc__ = """ +Return whether the argument is the [`TypeIs`][typing.TypeIs] [special form][]. + +```pycon +>>> is_typeis(TypeIs) +True +``` +""" + +_is_typealiastype_inner = _compile_isinstance_check_function('TypeAliasType', '_is_typealiastype_inner') + + +if _IS_PY310: + # Parameterized PEP 695 type aliases are instances of `types.GenericAlias` in typing_extensions>=4.13.0. + # On Python 3.10, with `Alias[int]` being such an instance of `GenericAlias`, + # `isinstance(Alias[int], TypeAliasType)` returns `True`. + # See https://github.com/python/cpython/issues/89828. + def is_typealiastype(obj: Any, /) -> 'TypeIs[TypeAliasType]': + return type(obj) is not GenericAlias and _is_typealiastype_inner(obj) +else: + is_typealiastype = _compile_isinstance_check_function('TypeAliasType', 'is_typealiastype') + +is_typealiastype.__doc__ = """ +Return whether the argument is a [`TypeAliasType`][typing.TypeAliasType] instance. + +```pycon +>>> type MyInt = int +>>> is_typealiastype(MyInt) +True +>>> MyStr = TypeAliasType("MyStr", str) +>>> is_typealiastype(MyStr): +True +>>> type MyList[T] = list[T] +>>> is_typealiastype(MyList[int]) +False +``` +""" + +is_unpack = _compile_identity_check_function('Unpack', 'is_unpack') +is_unpack.__doc__ = """ +Return whether the argument is the [`Unpack`][typing.Unpack] [special form][]. + +```pycon +>>> is_unpack(Unpack) +True +>>> is_unpack(Unpack[Ts]) +False +``` +""" + + +if sys.version_info >= (3, 13): + + def is_deprecated(obj: Any, /) -> 'TypeIs[deprecated]': + return isinstance(obj, (warnings.deprecated, typing_extensions.deprecated)) + +else: + + def is_deprecated(obj: Any, /) -> 'TypeIs[deprecated]': + return isinstance(obj, typing_extensions.deprecated) + + +is_deprecated.__doc__ = """ +Return whether the argument is a [`deprecated`][warnings.deprecated] instance. + +This also includes the [`typing_extensions` backport][typing_extensions.deprecated]. + +```pycon +>>> is_deprecated(warnings.deprecated('message')) +True +>>> is_deprecated(typing_extensions.deprecated('message')) +True +``` +""" + + +# Aliases defined in the `typing` module using `typing._SpecialGenericAlias` (itself aliased as `alias()`): +DEPRECATED_ALIASES: Final[dict[Any, type[Any]]] = { + typing.Hashable: collections.abc.Hashable, + typing.Awaitable: collections.abc.Awaitable, + typing.Coroutine: collections.abc.Coroutine, + typing.AsyncIterable: collections.abc.AsyncIterable, + typing.AsyncIterator: collections.abc.AsyncIterator, + typing.Iterable: collections.abc.Iterable, + typing.Iterator: collections.abc.Iterator, + typing.Reversible: collections.abc.Reversible, + typing.Sized: collections.abc.Sized, + typing.Container: collections.abc.Container, + typing.Collection: collections.abc.Collection, + # type ignore reason: https://github.com/python/typeshed/issues/6257: + typing.Callable: collections.abc.Callable, # pyright: ignore[reportAssignmentType, reportUnknownMemberType] + typing.AbstractSet: collections.abc.Set, + typing.MutableSet: collections.abc.MutableSet, + typing.Mapping: collections.abc.Mapping, + typing.MutableMapping: collections.abc.MutableMapping, + typing.Sequence: collections.abc.Sequence, + typing.MutableSequence: collections.abc.MutableSequence, + typing.Tuple: tuple, + typing.List: list, + typing.Deque: collections.deque, + typing.Set: set, + typing.FrozenSet: frozenset, + typing.MappingView: collections.abc.MappingView, + typing.KeysView: collections.abc.KeysView, + typing.ItemsView: collections.abc.ItemsView, + typing.ValuesView: collections.abc.ValuesView, + typing.Dict: dict, + typing.DefaultDict: collections.defaultdict, + typing.OrderedDict: collections.OrderedDict, + typing.Counter: collections.Counter, + typing.ChainMap: collections.ChainMap, + typing.Generator: collections.abc.Generator, + typing.AsyncGenerator: collections.abc.AsyncGenerator, + typing.Type: type, + # Defined in `typing.__getattr__`: + typing.Pattern: re.Pattern, + typing.Match: re.Match, + typing.ContextManager: contextlib.AbstractContextManager, + typing.AsyncContextManager: contextlib.AbstractAsyncContextManager, + # Skipped: `ByteString` (deprecated, removed in 3.14) +} +"""A mapping between the deprecated typing aliases to their replacement, as per [PEP 585](https://peps.python.org/pep-0585/).""" + + +# Add the `typing_extensions` aliases: +for alias, target in list(DEPRECATED_ALIASES.items()): + # Use `alias.__name__` when we drop support for Python 3.9 + if (te_alias := getattr(typing_extensions, alias._name, None)) is not None: + DEPRECATED_ALIASES[te_alias] = target diff --git a/lib/python3.12/site-packages/typing_inspection/typing_objects.pyi b/lib/python3.12/site-packages/typing_inspection/typing_objects.pyi new file mode 100644 index 0000000000000000000000000000000000000000..5071598005a21063ff3b2a2a1dedced885267de6 --- /dev/null +++ b/lib/python3.12/site-packages/typing_inspection/typing_objects.pyi @@ -0,0 +1,417 @@ +# Stub file generated using: +# `stubgen --inspect-mode --include-docstrings -m typing_inspection.typing_objects` +# (manual edits need to be applied). +"""Low-level introspection utilities for [`typing`][] members. + +The provided functions in this module check against both the [`typing`][] and [`typing_extensions`][] +variants, if they exists and are different. +""" + +import sys +from typing import Any, Final, ForwardRef, NewType, TypeVar + +from typing_extensions import ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeAliasType, TypeIs, TypeVarTuple, deprecated + +__all__ = [ + 'DEPRECATED_ALIASES', + 'NoneType', + 'is_annotated', + 'is_any', + 'is_classvar', + 'is_concatenate', + 'is_deprecated', + 'is_final', + 'is_generic', + 'is_literal', + 'is_literalstring', + 'is_namedtuple', + 'is_never', + 'is_newtype', + 'is_nodefault', + 'is_noextraitems', + 'is_noreturn', + 'is_notrequired', + 'is_paramspec', + 'is_paramspecargs', + 'is_paramspeckwargs', + 'is_readonly', + 'is_required', + 'is_self', + 'is_typealias', + 'is_typealiastype', + 'is_typeguard', + 'is_typeis', + 'is_typevar', + 'is_typevartuple', + 'is_union', + 'is_unpack', +] + +if sys.version_info >= (3, 10): + from types import NoneType +else: + NoneType = type(None) + +def is_annotated(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Annotated`][typing.Annotated] [special form][]. + + ```pycon + >>> is_annotated(Annotated) + True + >>> is_annotated(Annotated[int, ...]) + False + ``` + """ + +def is_any(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Any`][typing.Any] [special form][]. + + ```pycon + >>> is_any(Any) + True + ``` + """ + +def is_classvar(obj: Any, /) -> bool: + """ + Return whether the argument is the [`ClassVar`][typing.ClassVar] [type qualifier][]. + + ```pycon + >>> is_classvar(ClassVar) + True + >>> is_classvar(ClassVar[int]) + >>> False + ``` + """ + +def is_concatenate(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Concatenate`][typing.Concatenate] [special form][]. + + ```pycon + >>> is_concatenate(Concatenate) + True + >>> is_concatenate(Concatenate[int, P]) + False + ``` + """ + +def is_final(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Final`][typing.Final] [type qualifier][]. + + ```pycon + >>> is_final(Final) + True + >>> is_final(Final[int]) + False + ``` + """ + +def is_forwardref(obj: Any, /) -> TypeIs[ForwardRef]: + """ + Return whether the argument is an instance of [`ForwardRef`][typing.ForwardRef]. + + ```pycon + >>> is_forwardref(ForwardRef('T')) + True + ``` + """ + +def is_generic(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Generic`][typing.Generic] [special form][]. + + ```pycon + >>> is_generic(Generic) + True + >>> is_generic(Generic[T]) + False + ``` + """ + +def is_literal(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Literal`][typing.Literal] [special form][]. + + ```pycon + >>> is_literal(Literal) + True + >>> is_literal(Literal["a"]) + False + ``` + """ + +def is_paramspec(obj: Any, /) -> TypeIs[ParamSpec]: + """ + Return whether the argument is an instance of [`ParamSpec`][typing.ParamSpec]. + + ```pycon + >>> P = ParamSpec('P') + >>> is_paramspec(P) + True + ``` + """ + +def is_typevar(obj: Any, /) -> TypeIs[TypeVar]: + """ + Return whether the argument is an instance of [`TypeVar`][typing.TypeVar]. + + ```pycon + >>> T = TypeVar('T') + >>> is_typevar(T) + True + ``` + """ + +def is_typevartuple(obj: Any, /) -> TypeIs[TypeVarTuple]: + """ + Return whether the argument is an instance of [`TypeVarTuple`][typing.TypeVarTuple]. + + ```pycon + >>> Ts = TypeVarTuple('Ts') + >>> is_typevartuple(Ts) + True + ``` + """ + +def is_union(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Union`][typing.Union] [special form][]. + + This function can also be used to check for the [`Optional`][typing.Optional] [special form][], + as at runtime, `Optional[int]` is equivalent to `Union[int, None]`. + + ```pycon + >>> is_union(Union) + True + >>> is_union(Union[int, str]) + False + ``` + + !!! warning + This does not check for unions using the [new syntax][types-union] (e.g. `int | str`). + """ + +def is_namedtuple(obj: Any, /) -> bool: + """Return whether the argument is a named tuple type. + + This includes [`NamedTuple`][typing.NamedTuple] subclasses and classes created from the + [`collections.namedtuple`][] factory function. + + ```pycon + >>> class User(NamedTuple): + ... name: str + ... + >>> is_namedtuple(User) + True + >>> City = collections.namedtuple('City', []) + >>> is_namedtuple(City) + True + >>> is_namedtuple(NamedTuple) + False + ``` + """ + +def is_literalstring(obj: Any, /) -> bool: + """ + Return whether the argument is the [`LiteralString`][typing.LiteralString] [special form][]. + + ```pycon + >>> is_literalstring(LiteralString) + True + ``` + """ + +def is_never(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Never`][typing.Never] [special form][]. + + ```pycon + >>> is_never(Never) + True + ``` + """ + +def is_newtype(obj: Any, /) -> TypeIs[NewType]: + """ + Return whether the argument is a [`NewType`][typing.NewType]. + + ```pycon + >>> UserId = NewType("UserId", int) + >>> is_newtype(UserId) + True + ``` + """ + +def is_nodefault(obj: Any, /) -> bool: + """ + Return whether the argument is the [`NoDefault`][typing.NoDefault] sentinel object. + + ```pycon + >>> is_nodefault(NoDefault) + True + ``` + """ + +def is_noextraitems(obj: Any, /) -> bool: + """ + Return whether the argument is the `NoExtraItems` sentinel object. + + ```pycon + >>> is_noextraitems(NoExtraItems) + True + ``` + """ + +def is_noreturn(obj: Any, /) -> bool: + """ + Return whether the argument is the [`NoReturn`][typing.NoReturn] [special form][]. + + ```pycon + >>> is_noreturn(NoReturn) + True + >>> is_noreturn(Never) + False + ``` + """ + +def is_notrequired(obj: Any, /) -> bool: + """ + Return whether the argument is the [`NotRequired`][typing.NotRequired] [special form][]. + + ```pycon + >>> is_notrequired(NotRequired) + True + ``` + """ + +def is_paramspecargs(obj: Any, /) -> TypeIs[ParamSpecArgs]: + """ + Return whether the argument is an instance of [`ParamSpecArgs`][typing.ParamSpecArgs]. + + ```pycon + >>> P = ParamSpec('P') + >>> is_paramspecargs(P.args) + True + ``` + """ + +def is_paramspeckwargs(obj: Any, /) -> TypeIs[ParamSpecKwargs]: + """ + Return whether the argument is an instance of [`ParamSpecKwargs`][typing.ParamSpecKwargs]. + + ```pycon + >>> P = ParamSpec('P') + >>> is_paramspeckwargs(P.kwargs) + True + ``` + """ + +def is_readonly(obj: Any, /) -> bool: + """ + Return whether the argument is the [`ReadOnly`][typing.ReadOnly] [special form][]. + + ```pycon + >>> is_readonly(ReadOnly) + True + ``` + """ + +def is_required(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Required`][typing.Required] [special form][]. + + ```pycon + >>> is_required(Required) + True + ``` + """ + +def is_self(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Self`][typing.Self] [special form][]. + + ```pycon + >>> is_self(Self) + True + ``` + """ + +def is_typealias(obj: Any, /) -> bool: + """ + Return whether the argument is the [`TypeAlias`][typing.TypeAlias] [special form][]. + + ```pycon + >>> is_typealias(TypeAlias) + True + ``` + """ + +def is_typeguard(obj: Any, /) -> bool: + """ + Return whether the argument is the [`TypeGuard`][typing.TypeGuard] [special form][]. + + ```pycon + >>> is_typeguard(TypeGuard) + True + ``` + """ + +def is_typeis(obj: Any, /) -> bool: + """ + Return whether the argument is the [`TypeIs`][typing.TypeIs] [special form][]. + + ```pycon + >>> is_typeis(TypeIs) + True + ``` + """ + +def is_typealiastype(obj: Any, /) -> TypeIs[TypeAliasType]: + """ + Return whether the argument is a [`TypeAliasType`][typing.TypeAliasType] instance. + + ```pycon + >>> type MyInt = int + >>> is_typealiastype(MyInt) + True + >>> MyStr = TypeAliasType("MyStr", str) + >>> is_typealiastype(MyStr): + True + >>> type MyList[T] = list[T] + >>> is_typealiastype(MyList[int]) + False + ``` + """ + +def is_unpack(obj: Any, /) -> bool: + """ + Return whether the argument is the [`Unpack`][typing.Unpack] [special form][]. + + ```pycon + >>> is_unpack(Unpack) + True + >>> is_unpack(Unpack[Ts]) + False + ``` + """ + +def is_deprecated(obj: Any, /) -> TypeIs[deprecated]: + """ + Return whether the argument is a [`deprecated`][warnings.deprecated] instance. + + This also includes the [`typing_extensions` backport][typing_extensions.deprecated]. + + ```pycon + >>> is_deprecated(warnings.deprecated('message')) + True + >>> is_deprecated(typing_extensions.deprecated('deprecated')) + True + ``` + """ + +DEPRECATED_ALIASES: Final[dict[Any, type[Any]]] +"""A mapping between the deprecated typing aliases to their replacement, as per [PEP 585](https://peps.python.org/pep-0585/)."""